
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf

# Elu code
def elu (x, alpha):
    return np.where(x > 0,x, alpha*(np.exp(x)-1))

def elu_derivative(x, alpha):
    return np.where(x > 0, 1, alpha*np.exp(x))

# Plotting code
x = np.linspace(-5, 5, 400)
alpha = 0.1
elu_values = elu(x, alpha)
elu_derivative_values = elu_derivative(x, alpha)

plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 2)
plt.plot(x, elu_values, label="ELU")
plt.plot(x, elu_derivative_values, label="ELU's Derivative")
plt.title("ELU and Derivative Values")
plt.xlabel('x')
plt.ylabel("f'(x)")
plt.legend()

plt.tight_layout()
plt.show()


#Gelu code
def gelu(x):
    cdf = 0.5 * (1.0 + tf.math.erf(x / tf.sqrt(2.0)))
    return x * cdf

def gelu_derivative(x):
    with tf.GradientTape() as tape:
        tape.watch(x)
        y = gelu(x)
    return tape.gradient(y, x)

#Plotting code
x = np.linspace(-5, 5, 400)
x_tf = tf.constant(x, dtype=tf.float32)
gelu_values = gelu(x_tf)
gelu_derivative_values = gelu_derivative(x_tf)

plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 2)
plt.plot(x, gelu_values, label="GELU")
plt.plot(x, gelu_derivative_values, label="GELU's Derivative")
plt.title("GELU and Derivative Values")
plt.xlabel('x')
plt.ylabel("f'(x)")
plt.legend()

plt.tight_layout()
plt.show()


import numpy as np


# Answers for Questions
x = np.matrix([[1],[-1],[-2]])

w1 = np.matrix([[1,-1,2],[-1,0.5,1],[-2,1,2],[0,0,1]])
w2 = np.matrix([[-1,1,0,1],[1,1,0,-2],[0.5,-1,2,0]])
w3 = np.matrix ([[1,-2,0],[0,2,0],[1,-1,1],[0.5,1,2]])

b1 = np.matrix ([[1],[0],[1],[0]])
b2 = np.matrix ([[0],[0.5],[1]])
b3 = np.matrix ([[-1],[1],[-1],[1]])

# h1
a1 = np.maximum(np.dot(w1, x) + b1, 0)
#h2
a2 = np.maximum(np.dot(w2, a1) + b2, 0)
#h3 logit
a3 = np.dot(w3, a2) + b3

print ("a)\n")
print (a1,"\n")
print ("b)\n")
print (a2,"\n")
print ("c)\n")
print (a3,"\n")

def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0)

#prediction probability
prediction_probability = softmax(a3)
                                 
print ("d)\n")
print (prediction_probability,"\n")
print ("e)\n")
#prediction label
predicted_label = np.argmax(prediction_probability)
print ("Predicted label = ",predicted_label + 1,"\n")
print ("f)\n")
one_hot = np.zeros(prediction_probability.shape,dtype=int)
one_hot[predicted_label] = 1
target_index = 3
#ce loss
ce_loss = -np.log(prediction_probability[target_index-1])
print ("Cross entropy loss =",ce_loss[0,0],"\n")
print ("g)\n")

a)

[[0.]
 [0.]
 [0.]
 [0.]] 

b)

[[0. ]
 [0.5]
 [1. ]] 

c)

[[-2. ]
 [ 2. ]
 [-0.5]
 [ 3.5]] 

d)

[[0.00328114]
 [0.17914438]
 [0.01470507]
 [0.80286941]] 

e)

Predicted label =  4 

f)

Cross entropy loss = 4.219563205900562 

g)


#Code to generate random matrices and biases for W1, b1, W2, b2
import numpy as np
student_id = 31237223           #insert your student id here for example 1234    
np.random.seed(student_id)
W1 = np.random.rand(5,3)
b1 = np.random.rand(5,1)
W2 = np.random.rand(3,5)
b2 = np.random.rand(3,1)


#Show your code


#Show your code


#Show your code


#Code to generate random matrices and biases for W1, b1, W2, b2
import numpy as np
student_id = 31237223 #insert your student id here for example 1234    
np.random.seed(student_id)
W1 = np.random.rand(5,3)
b1 = np.random.rand(5,1)
W2 = np.random.rand(3,5)
b2 = np.random.rand(3,1)


# Show your code
def elu(x, alpha=0.1):
    """ELU activation function."""
    y = np.where(x > 0, x, alpha * (np.exp(x) - 1))
    return y

# Given input examples
x1 = np.matrix([ 1, -1,  1]).T
x2 = np.matrix([-1,  2, -1]).T
x3 = np.matrix([-1.5, 1,  0]).T
x4 = np.matrix([-1,  2, -1]).T
x5 = np.matrix([ 0,  2.5, 1.5]).T

# Stack the input examples to create a mini-batch matrix
mini_batch_matrix = np.hstack((x1, x2, x3, x4, x5))

qA = np.dot(W1,mini_batch_matrix) + b1
print ("answer:\n",qA)

answer:
 [[ 1.01009806  0.30364112 -0.17987178  0.30364112  2.93026733]
 [ 0.45984992  1.08805968  0.05061028  1.08805968  2.59282736]
 [ 1.11497155  1.27251212  0.33600097  1.27251212  3.0365595 ]
 [ 1.27415222  1.2745985   1.22620037  1.2745985   3.40831993]
 [ 0.64008934  0.37189129 -0.47838965  0.37189129  1.50595585]]


#Show your code
qB = elu(qA)
print ("answer:\n",qB)

answer:
 [[ 1.01009806  0.30364112 -0.01646227  0.30364112  2.93026733]
 [ 0.45984992  1.08805968  0.05061028  1.08805968  2.59282736]
 [ 1.11497155  1.27251212  0.33600097  1.27251212  3.0365595 ]
 [ 1.27415222  1.2745985   1.22620037  1.2745985   3.40831993]
 [ 0.64008934  0.37189129 -0.03802193  0.37189129  1.50595585]]


#Show your code
# correct solution:
qC = np.dot(W2,qB) + b2

def softmax(x):
    """Compute softmax values for each sets of scores in x."""
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0) 

# Get softmax values
softmax_qC = softmax(qC)
prediction_qC = np.argmax(softmax(qC),axis=0)

# init one hot array
one_hot = np.zeros(softmax_qC.shape,dtype=int)
prediction_qC = np.expand_dims(prediction_qC, 0)

# create new one hot array based on prediction
np.put_along_axis(one_hot,prediction_qC,1,axis=0)

print ("softmax:\n",softmax_qC)
print ("one hot:\n",one_hot)
print ("prediction value\n",prediction_qC)

softmax:
 [[0.55341198 0.43520274 0.29568302 0.43520274 0.91412831]
 [0.23225884 0.20552616 0.25346639 0.20552616 0.03970293]
 [0.21432918 0.3592711  0.45085058 0.3592711  0.04616876]]
one hot:
 [[1 1 0 1 1]
 [0 0 0 0 0]
 [0 0 1 0 0]]
prediction value
 [[0 0 2 0 0]]


#Show your code

# original target array = [2,1,3,1,2], scaled to fit array index
one_hot_target = np.zeros(softmax_qC.shape,dtype=int)
target = [1,0,2,0,1]
target = np.expand_dims(target, 0)
np.put_along_axis(one_hot_target,target,1,axis=0)
ce = - np.sum(one_hot_target * np.log(softmax_qC)) / 5
print ("ce loss:\n",ce)

ce loss:
 1.4293477795659966


#Part 1:
p1 = softmax_qC - one_hot_target
print ("Part 1:\n",p1)
p2 = np.dot(p1,qB.T)
print ("\nPart 2:\n",p2)
p3 = p1.T
print ("\nPart 3:\n",p3)
matr = np.sum (p1.T,axis = 0)
print (matr.reshape(1, -1).T)

Part 1:
 [[ 0.55341198 -0.56479726  0.29568302 -0.56479726  0.91412831]
 [-0.76774116  0.20552616  0.25346639  0.20552616 -0.96029707]
 [ 0.21432918  0.3592711  -0.54914942  0.3592711   0.04616876]]

Part 2:
 [[ 2.88978173  1.41056169  2.05477067  2.74355998  1.29954118]
 [-3.46878122 -2.38285275 -3.16377471 -3.41649147 -1.79435842]
 [ 0.57899949  0.97229106  1.10900404  0.6729315   0.49481725]]

Part 3:
 [[ 0.55341198 -0.76774116  0.21432918]
 [-0.56479726  0.20552616  0.3592711 ]
 [ 0.29568302  0.25346639 -0.54914942]
 [-0.56479726  0.20552616  0.3592711 ]
 [ 0.91412831 -0.96029707  0.04616876]]
[[ 0.63362879]
 [-1.06351951]
 [ 0.42989072]]


#Part 1

p1_f = np.dot (p1.T,W2)
print (p1_f)

def elu_derivative(x, alpha):
    return np.where(x > 0, 1, alpha*np.exp(x))

p2_f = np.dot (p1_f,elu_derivative(qA,0.1))

print (p2_f)

p3_f = np.dot (p2_f,mini_batch_matrix.T)

print (p3_f)

p4_f = p2_f.T

print (p4_f)

[[-0.19335577  0.0518627   0.13949576  0.42682301  0.44642821]
 [-0.23272077  0.05467927 -0.14588584 -0.09308672 -0.44906295]
 [ 0.39047495 -0.09584548  0.07857322 -0.16522643  0.23100242]
 [-0.23272077  0.05467927 -0.14588584 -0.09308672 -0.44906295]
 [-0.09032912  0.02835206  0.23229467  0.52259559  0.7339236 ]]
[[ 0.87125391  0.87125391  0.62969772  0.87125391  0.87125391]
 [-0.866077   -0.866077   -0.2315663  -0.866077   -0.866077  ]
 [ 0.43897868  0.43897868 -0.13556222  0.43897868  0.43897868]
 [-0.866077   -0.866077   -0.2315663  -0.866077   -0.866077  ]
 [ 1.4268368   1.4268368   0.82118359  1.4268368   1.4268368 ]]
[[-1.8158005   5.42159424  0.43562696]
 [ 1.21342645 -4.9949898  -0.4330385 ]
 [-0.23563535  2.27882052  0.21948934]
 [ 1.21342645 -4.9949898  -0.4330385 ]
 [-2.6586122   8.66878602  0.7134184 ]]
[[ 0.87125391 -0.866077    0.43897868 -0.866077    1.4268368 ]
 [ 0.87125391 -0.866077    0.43897868 -0.866077    1.4268368 ]
 [ 0.62969772 -0.2315663  -0.13556222 -0.2315663   0.82118359]
 [ 0.87125391 -0.866077    0.43897868 -0.866077    1.4268368 ]
 [ 0.87125391 -0.866077    0.43897868 -0.866077    1.4268368 ]]


#Show your code
learning_rate = 0.01

# Update W2 and b2 using SGD
W2 -= learning_rate * p2
b2 -= learning_rate * matr.reshape(1, -1).T

print ("W2:\n",W2,"\n","b2:\n",b2)

# Update W1 and b1 using SGD
W1 -= learning_rate * p3_f
# b1 -= learning_rate * np.sum(p4_f)

print ("W1:\n",W1,"\n","b1:\n",b1)

W2:
 [[0.729301   0.50671945 0.87949781 0.79949605 0.97970768]
 [0.85412348 0.52304093 0.67725963 0.31933795 0.2083518 ]
 [0.06961929 0.675661   0.62844156 0.87102426 0.19679131]] 
 b2:
 [[0.11045054]
 [0.69475357]
 [0.50990836]]
W1:
 [[0.80534526 0.6338631  0.59380351]
 [0.73795092 0.85351641 0.14549009]
 [0.82473595 0.72886885 0.22414081]
 [0.21397833 0.6319377  0.65097635]
 [0.84134583 0.38883133 0.02548392]] 
 b1:
 [[0.31283007]
 [0.37217154]
 [0.81791331]
 [0.98338146]
 [0.26823072]]


import tensorflow as tf
from tensorflow import keras


tf.random.set_seed(1234)


fashion_mnist = keras.datasets.fashion_mnist
(X_train_full_img, y_train_full), (X_test_img, y_test) = fashion_mnist.load_data()


num_train = X_train_full_img.shape[0]
num_test = X_test_img.shape[0]

#Get X_train_full and X_test
X_train_full =  X_train_full_img.reshape(num_train,-1)
X_test =   X_test_img.reshape(num_test, -1)

#Print shape of test and train set
print("train set shape:\n",X_train_full.shape, y_train_full.shape)
print("test set shape: \n",X_test.shape, y_test.shape)

train set shape:
 (60000, 784) (60000,)
test set shape: 
 (10000, 784) (10000,)


import math
N = X_train_full.shape[0]
i = math.floor(0.9*N)
n_classes= 10

shuffle = np.random.permutation(N)

#split and shuffle the dataset according to method taught in the tutorials 
valid_idx = math.floor(0.1*N)
X_train, y_train = X_train_full[shuffle][:i],y_train_full[shuffle][:i]
X_valid, y_valid = X_train_full[shuffle][i:i+valid_idx],y_train_full[shuffle][i:i+valid_idx]
X_train, X_valid, X_test = X_train/255.0 , X_valid/255.0 , X_test/255.0

print ('Train set', X_train.shape, y_train.shape)
print('Validation set', X_valid.shape, y_valid.shape)
print('Test set', X_test.shape, y_test.shape)

Train set (54000, 784) (54000,)
Validation set (6000, 784) (6000,)
Test set (10000, 784) (10000,)


# Code adapted from tutorials
class DNN:
    def __init__(self,n1,n2,act,n_classes=10, optimizer=tf.keras.optimizers.SGD(learning_rate=0.01),
                 batch_size=32, epochs=1, alpha=0.001):
        self.n_classes = n_classes
        self.batch_size = batch_size
        self.epochs = epochs
        self.optimizer = optimizer
        self.alpha = alpha 
        self.n1 = n1
        self.n2 = n2
        self.act = act

        # create a tensorflow dataset for training
        self.train_set = tf.data.Dataset.from_tensor_slices((X_train, y_train))
        # create a tensorflow dataset for validation
        self.valid_set = tf.data.Dataset.from_tensor_slices((X_valid, y_valid))
        # create a tensorflow dataset for validation
        self.test_set = tf.data.Dataset.from_tensor_slices((X_test, y_test))
        # batching train and valid sets
        self.train_set = self.train_set.batch(self.batch_size).prefetch(1)
        self.valid_set = self.valid_set.batch(self.batch_size).prefetch(1)
        self.test_set = self.test_set.batch(self.batch_size).prefetch(1)
        tf.keras.backend.set_floatx('float64')


    def build(self):
        self.model = tf.keras.Sequential([
            tf.keras.layers.Dense(self.n1, activation= self.act),
            tf.keras.layers.Dense(self.n2, activation= self.act),
            tf.keras.layers.Dense(self.n_classes, activation='softmax')
        ])

    def compute_loss(self, X, y):  # X is data batch, y is label batch
        pred_probs = self.model(X)
        l1 = tf.keras.losses.sparse_categorical_crossentropy(y, pred_probs)  # Cross entropy loss
        l2 = tf.add_n([tf.nn.l2_loss(w) for w in self.model.trainable_weights])
        l2 = tf.expand_dims(l2, axis=-1)
        return l1 + self.alpha * l2


    def compute_grads(self, X, y):
        with tf.GradientTape() as g:  # use gradient tape to compute gradients
            loss = self.compute_loss(X, y)
        grads = g.gradient(loss, self.model.trainable_variables)  # compute gradients w.r.t. all trainable variables
        return grads

    def train_one_batch(self, X, y):  # train in one batch
        grads = self.compute_grads(X, y)
        # the gradients will be applied according to optimizer for example SGD, Adam, and etc.
        self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables))

    def evaluate(self, tf_dataset=None):
        dataset_loss = tf.keras.metrics.Mean()
        dataset_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
        for X, y in tf_dataset:
            loss = self.compute_loss(X, y)
            dataset_loss.update_state(loss)
            dataset_accuracy.update_state(y, self.model(X, training=False))
        return dataset_loss.result(), dataset_accuracy.result()

    def train(self):
        for epoch in range(self.epochs):
            for X, y in self.train_set:  # use batch_index if you want to display something in iterations
                self.train_one_batch(X, y)
            train_loss, train_acc = self.evaluate(self.train_set)
            valid_loss, valid_acc = self.evaluate(self.valid_set)
            print('Epoch {}: train acc= {:.4f}, train loss= {:.4f} | valid acc= {:.4f}, valid loss= {:.4f}'.format(
                epoch + 1, train_acc, train_loss, valid_acc, valid_loss))
        return valid_acc,valid_loss
    
    def evaluate_test_set(self, tf_dataset=None):
        dataset_loss = tf.keras.metrics.Mean()
        dataset_accuracy = tf.keras.metrics.SparseCategoricalAccuracy()
        for X, y in self.test_set:
            loss = self.compute_loss(X, y)
            dataset_loss.update_state(loss)
            dataset_accuracy.update_state(y, self.model(X, training=False))
        return ('test acc = {:.4f}, test loss = {:.4f}'.format(
                dataset_accuracy.result().numpy(), dataset_loss.result().numpy()))
    
opt = tf.keras.optimizers.Adam()
dnn = DNN(40,30,'relu',optimizer=opt, epochs=50, batch_size=64)
dnn.build()
dnn.train()
dnn.evaluate_test_set()

Epoch 1: train acc= 0.8448, train loss= 0.5111 | valid acc= 0.8433, valid loss= 0.5197
Epoch 2: train acc= 0.8578, train loss= 0.4694 | valid acc= 0.8518, valid loss= 0.4884
Epoch 3: train acc= 0.8651, train loss= 0.4498 | valid acc= 0.8565, valid loss= 0.4749
Epoch 4: train acc= 0.8698, train loss= 0.4372 | valid acc= 0.8622, valid loss= 0.4687
Epoch 5: train acc= 0.8728, train loss= 0.4290 | valid acc= 0.8648, valid loss= 0.4646
Epoch 6: train acc= 0.8756, train loss= 0.4203 | valid acc= 0.8663, valid loss= 0.4598
Epoch 7: train acc= 0.8780, train loss= 0.4150 | valid acc= 0.8672, valid loss= 0.4557
Epoch 8: train acc= 0.8811, train loss= 0.4090 | valid acc= 0.8693, valid loss= 0.4532
Epoch 9: train acc= 0.8834, train loss= 0.4041 | valid acc= 0.8707, valid loss= 0.4501
Epoch 10: train acc= 0.8849, train loss= 0.4018 | valid acc= 0.8690, valid loss= 0.4492
Epoch 11: train acc= 0.8873, train loss= 0.3955 | valid acc= 0.8723, valid loss= 0.4443
Epoch 12: train acc= 0.8887, train loss= 0.3925 | valid acc= 0.8720, valid loss= 0.4431
Epoch 13: train acc= 0.8899, train loss= 0.3911 | valid acc= 0.8723, valid loss= 0.4440
Epoch 14: train acc= 0.8918, train loss= 0.3861 | valid acc= 0.8757, valid loss= 0.4394
Epoch 15: train acc= 0.8921, train loss= 0.3862 | valid acc= 0.8738, valid loss= 0.4398
Epoch 16: train acc= 0.8929, train loss= 0.3834 | valid acc= 0.8745, valid loss= 0.4371
Epoch 17: train acc= 0.8921, train loss= 0.3847 | valid acc= 0.8722, valid loss= 0.4393
Epoch 18: train acc= 0.8922, train loss= 0.3847 | valid acc= 0.8745, valid loss= 0.4420
Epoch 19: train acc= 0.8940, train loss= 0.3799 | valid acc= 0.8755, valid loss= 0.4372
Epoch 20: train acc= 0.8936, train loss= 0.3818 | valid acc= 0.8743, valid loss= 0.4398
Epoch 21: train acc= 0.8925, train loss= 0.3861 | valid acc= 0.8732, valid loss= 0.4425
Epoch 22: train acc= 0.8932, train loss= 0.3828 | valid acc= 0.8722, valid loss= 0.4410
Epoch 23: train acc= 0.8943, train loss= 0.3817 | valid acc= 0.8737, valid loss= 0.4413
Epoch 24: train acc= 0.8942, train loss= 0.3820 | valid acc= 0.8728, valid loss= 0.4422
Epoch 25: train acc= 0.8960, train loss= 0.3772 | valid acc= 0.8747, valid loss= 0.4386
Epoch 26: train acc= 0.8963, train loss= 0.3771 | valid acc= 0.8725, valid loss= 0.4370
Epoch 27: train acc= 0.8964, train loss= 0.3754 | valid acc= 0.8733, valid loss= 0.4376
Epoch 28: train acc= 0.8956, train loss= 0.3781 | valid acc= 0.8717, valid loss= 0.4388
Epoch 29: train acc= 0.8966, train loss= 0.3741 | valid acc= 0.8742, valid loss= 0.4360
Epoch 30: train acc= 0.8960, train loss= 0.3758 | valid acc= 0.8753, valid loss= 0.4384
Epoch 31: train acc= 0.8949, train loss= 0.3793 | valid acc= 0.8742, valid loss= 0.4425
Epoch 32: train acc= 0.8957, train loss= 0.3771 | valid acc= 0.8737, valid loss= 0.4409
Epoch 33: train acc= 0.8982, train loss= 0.3739 | valid acc= 0.8725, valid loss= 0.4397
Epoch 34: train acc= 0.8965, train loss= 0.3753 | valid acc= 0.8740, valid loss= 0.4408
Epoch 35: train acc= 0.8964, train loss= 0.3760 | valid acc= 0.8732, valid loss= 0.4419
Epoch 36: train acc= 0.8979, train loss= 0.3733 | valid acc= 0.8753, valid loss= 0.4400
Epoch 37: train acc= 0.8978, train loss= 0.3753 | valid acc= 0.8723, valid loss= 0.4420
Epoch 38: train acc= 0.8960, train loss= 0.3793 | valid acc= 0.8735, valid loss= 0.4455
Epoch 39: train acc= 0.8985, train loss= 0.3732 | valid acc= 0.8742, valid loss= 0.4417
Epoch 40: train acc= 0.8986, train loss= 0.3735 | valid acc= 0.8742, valid loss= 0.4427
Epoch 41: train acc= 0.8984, train loss= 0.3719 | valid acc= 0.8762, valid loss= 0.4397
Epoch 42: train acc= 0.8995, train loss= 0.3694 | valid acc= 0.8757, valid loss= 0.4373
Epoch 43: train acc= 0.8996, train loss= 0.3710 | valid acc= 0.8753, valid loss= 0.4415
Epoch 44: train acc= 0.8998, train loss= 0.3714 | valid acc= 0.8752, valid loss= 0.4422
Epoch 45: train acc= 0.9011, train loss= 0.3692 | valid acc= 0.8760, valid loss= 0.4396
Epoch 46: train acc= 0.9007, train loss= 0.3698 | valid acc= 0.8770, valid loss= 0.4419
Epoch 47: train acc= 0.9011, train loss= 0.3684 | valid acc= 0.8748, valid loss= 0.4407
Epoch 48: train acc= 0.9016, train loss= 0.3681 | valid acc= 0.8780, valid loss= 0.4394
Epoch 49: train acc= 0.9010, train loss= 0.3707 | valid acc= 0.8752, valid loss= 0.4425
Epoch 50: train acc= 0.9019, train loss= 0.3674 | valid acc= 0.8765, valid loss= 0.4411

'test acc = 0.8721, test loss = 0.4527'


# Perform grid search using each possible configurtaions, compute 10 epochs and save the best values / configuration
lst_n1 = [20,40]
lst_n2 = [20,40]
lst_activation = ['sigmoid','tanh','relu']
best_acc= - np.inf
best_history = None
best_model = None
for n1 in lst_n1:
    for n2 in lst_n2:
        for act in lst_activation:
            print('\nThis is for model n1= {}, n2 = {}, activation function = {}'.format(n1,n2, act))
            opt = tf.keras.optimizers.SGD(learning_rate=0.001)
            dnn = DNN(n1,n2,act,optimizer=opt, epochs=10, batch_size=64)
            dnn.build()
            valid_acc, valid_loss = dnn.train()
            print('\tvalid acc = {}, valid loss = {}'.format(valid_acc, valid_loss))
            if(valid_acc > best_acc):
                best_model = dnn
                best_acc = valid_acc
                best_n1 = n1
                best_n2 = n2
                best_act = act

print('\nThe best model is with n1= {}, n2 = {}, activation function = {}'.format(best_n1,best_n2, best_act))

This is for model n1= 20, n2 = 20, activation function = sigmoid
Epoch 1: train acc= 0.4086, train loss= 1.9378 | valid acc= 0.4068, valid loss= 1.9373
Epoch 2: train acc= 0.5539, train loss= 1.7302 | valid acc= 0.5563, valid loss= 1.7289
Epoch 3: train acc= 0.5677, train loss= 1.6562 | valid acc= 0.5697, valid loss= 1.6547
Epoch 4: train acc= 0.5812, train loss= 1.6286 | valid acc= 0.5822, valid loss= 1.6269
Epoch 5: train acc= 0.5960, train loss= 1.6133 | valid acc= 0.5995, valid loss= 1.6115
Epoch 6: train acc= 0.6110, train loss= 1.6030 | valid acc= 0.6090, valid loss= 1.6011
Epoch 7: train acc= 0.6218, train loss= 1.5955 | valid acc= 0.6195, valid loss= 1.5935
Epoch 8: train acc= 0.6276, train loss= 1.5899 | valid acc= 0.6243, valid loss= 1.5878
Epoch 9: train acc= 0.6314, train loss= 1.5858 | valid acc= 0.6273, valid loss= 1.5837
Epoch 10: train acc= 0.6340, train loss= 1.5828 | valid acc= 0.6305, valid loss= 1.5806
	valid acc = 0.6305, valid loss = 1.5805526927139606

This is for model n1= 20, n2 = 20, activation function = tanh
Epoch 1: train acc= 0.8189, train loss= 0.8983 | valid acc= 0.8262, valid loss= 0.8904
Epoch 2: train acc= 0.8326, train loss= 0.8179 | valid acc= 0.8373, valid loss= 0.8120
Epoch 3: train acc= 0.8372, train loss= 0.7907 | valid acc= 0.8422, valid loss= 0.7857
Epoch 4: train acc= 0.8400, train loss= 0.7792 | valid acc= 0.8445, valid loss= 0.7746
Epoch 5: train acc= 0.8412, train loss= 0.7733 | valid acc= 0.8455, valid loss= 0.7689
Epoch 6: train acc= 0.8424, train loss= 0.7697 | valid acc= 0.8468, valid loss= 0.7655
Epoch 7: train acc= 0.8426, train loss= 0.7674 | valid acc= 0.8472, valid loss= 0.7633
Epoch 8: train acc= 0.8428, train loss= 0.7657 | valid acc= 0.8482, valid loss= 0.7617
Epoch 9: train acc= 0.8434, train loss= 0.7644 | valid acc= 0.8480, valid loss= 0.7605
Epoch 10: train acc= 0.8436, train loss= 0.7634 | valid acc= 0.8483, valid loss= 0.7595
	valid acc = 0.8483333333333334, valid loss = 0.75954389014383

This is for model n1= 20, n2 = 20, activation function = relu
Epoch 1: train acc= 0.8170, train loss= 0.8341 | valid acc= 0.8268, valid loss= 0.8276
Epoch 2: train acc= 0.8306, train loss= 0.7501 | valid acc= 0.8380, valid loss= 0.7463
Epoch 3: train acc= 0.8346, train loss= 0.7235 | valid acc= 0.8392, valid loss= 0.7208
Epoch 4: train acc= 0.8372, train loss= 0.7125 | valid acc= 0.8390, valid loss= 0.7106
Epoch 5: train acc= 0.8389, train loss= 0.7064 | valid acc= 0.8413, valid loss= 0.7046
Epoch 6: train acc= 0.8394, train loss= 0.7040 | valid acc= 0.8418, valid loss= 0.7023
Epoch 7: train acc= 0.8397, train loss= 0.7021 | valid acc= 0.8438, valid loss= 0.7004
Epoch 8: train acc= 0.8396, train loss= 0.7016 | valid acc= 0.8447, valid loss= 0.6999
Epoch 9: train acc= 0.8402, train loss= 0.7012 | valid acc= 0.8432, valid loss= 0.6995
Epoch 10: train acc= 0.8412, train loss= 0.6974 | valid acc= 0.8462, valid loss= 0.6956
	valid acc = 0.8461666666666666, valid loss = 0.6956203009179754

This is for model n1= 20, n2 = 40, activation function = sigmoid
Epoch 1: train acc= 0.4802, train loss= 1.9050 | valid acc= 0.4842, valid loss= 1.9036
Epoch 2: train acc= 0.5258, train loss= 1.6948 | valid acc= 0.5308, valid loss= 1.6902
Epoch 3: train acc= 0.5655, train loss= 1.6388 | valid acc= 0.5692, valid loss= 1.6338
Epoch 4: train acc= 0.5841, train loss= 1.6130 | valid acc= 0.5882, valid loss= 1.6082
Epoch 5: train acc= 0.5939, train loss= 1.5965 | valid acc= 0.5987, valid loss= 1.5924
Epoch 6: train acc= 0.6021, train loss= 1.5845 | valid acc= 0.6070, valid loss= 1.5807
Epoch 7: train acc= 0.6102, train loss= 1.5751 | valid acc= 0.6158, valid loss= 1.5717
Epoch 8: train acc= 0.6185, train loss= 1.5659 | valid acc= 0.6202, valid loss= 1.5628
Epoch 9: train acc= 0.6275, train loss= 1.5558 | valid acc= 0.6260, valid loss= 1.5529
Epoch 10: train acc= 0.6371, train loss= 1.5455 | valid acc= 0.6362, valid loss= 1.5426
	valid acc = 0.6361666666666667, valid loss = 1.542632225107603

This is for model n1= 20, n2 = 40, activation function = tanh
Epoch 1: train acc= 0.8239, train loss= 0.8762 | valid acc= 0.8340, valid loss= 0.8689
Epoch 2: train acc= 0.8367, train loss= 0.7930 | valid acc= 0.8423, valid loss= 0.7863
Epoch 3: train acc= 0.8410, train loss= 0.7651 | valid acc= 0.8448, valid loss= 0.7587
Epoch 4: train acc= 0.8432, train loss= 0.7536 | valid acc= 0.8470, valid loss= 0.7473
Epoch 5: train acc= 0.8439, train loss= 0.7478 | valid acc= 0.8490, valid loss= 0.7416
Epoch 6: train acc= 0.8442, train loss= 0.7443 | valid acc= 0.8497, valid loss= 0.7383
Epoch 7: train acc= 0.8444, train loss= 0.7420 | valid acc= 0.8510, valid loss= 0.7361
Epoch 8: train acc= 0.8448, train loss= 0.7402 | valid acc= 0.8512, valid loss= 0.7345
Epoch 9: train acc= 0.8451, train loss= 0.7388 | valid acc= 0.8513, valid loss= 0.7332
Epoch 10: train acc= 0.8452, train loss= 0.7377 | valid acc= 0.8510, valid loss= 0.7321
	valid acc = 0.851, valid loss = 0.7321288660559229

This is for model n1= 20, n2 = 40, activation function = relu
Epoch 1: train acc= 0.8120, train loss= 0.8586 | valid acc= 0.8227, valid loss= 0.8473
Epoch 2: train acc= 0.8259, train loss= 0.7630 | valid acc= 0.8370, valid loss= 0.7529
Epoch 3: train acc= 0.8310, train loss= 0.7307 | valid acc= 0.8427, valid loss= 0.7214
Epoch 4: train acc= 0.8342, train loss= 0.7166 | valid acc= 0.8443, valid loss= 0.7083
Epoch 5: train acc= 0.8370, train loss= 0.7090 | valid acc= 0.8460, valid loss= 0.7018
Epoch 6: train acc= 0.8390, train loss= 0.7048 | valid acc= 0.8467, valid loss= 0.6986
Epoch 7: train acc= 0.8404, train loss= 0.7022 | valid acc= 0.8463, valid loss= 0.6969
Epoch 8: train acc= 0.8414, train loss= 0.7000 | valid acc= 0.8475, valid loss= 0.6954
Epoch 9: train acc= 0.8422, train loss= 0.6988 | valid acc= 0.8475, valid loss= 0.6947
Epoch 10: train acc= 0.8428, train loss= 0.6977 | valid acc= 0.8463, valid loss= 0.6940
	valid acc = 0.8463333333333334, valid loss = 0.6939578115844145

This is for model n1= 40, n2 = 20, activation function = sigmoid
Epoch 1: train acc= 0.5335, train loss= 1.8716 | valid acc= 0.5398, valid loss= 1.8687
Epoch 2: train acc= 0.5695, train loss= 1.6742 | valid acc= 0.5695, valid loss= 1.6713
Epoch 3: train acc= 0.5835, train loss= 1.6071 | valid acc= 0.5860, valid loss= 1.6042
Epoch 4: train acc= 0.5970, train loss= 1.5741 | valid acc= 0.6035, valid loss= 1.5710
Epoch 5: train acc= 0.6095, train loss= 1.5544 | valid acc= 0.6120, valid loss= 1.5511
Epoch 6: train acc= 0.6196, train loss= 1.5409 | valid acc= 0.6190, valid loss= 1.5375
Epoch 7: train acc= 0.6263, train loss= 1.5311 | valid acc= 0.6290, valid loss= 1.5276
Epoch 8: train acc= 0.6333, train loss= 1.5241 | valid acc= 0.6318, valid loss= 1.5206
Epoch 9: train acc= 0.6401, train loss= 1.5193 | valid acc= 0.6372, valid loss= 1.5158
Epoch 10: train acc= 0.6492, train loss= 1.5159 | valid acc= 0.6450, valid loss= 1.5124
	valid acc = 0.645, valid loss = 1.5124126107648554

This is for model n1= 40, n2 = 20, activation function = tanh
Epoch 1: train acc= 0.8199, train loss= 0.9488 | valid acc= 0.8318, valid loss= 0.9380
Epoch 2: train acc= 0.8319, train loss= 0.8268 | valid acc= 0.8392, valid loss= 0.8179
Epoch 3: train acc= 0.8369, train loss= 0.7848 | valid acc= 0.8425, valid loss= 0.7772
Epoch 4: train acc= 0.8397, train loss= 0.7686 | valid acc= 0.8455, valid loss= 0.7619
Epoch 5: train acc= 0.8416, train loss= 0.7615 | valid acc= 0.8470, valid loss= 0.7554
Epoch 6: train acc= 0.8431, train loss= 0.7577 | valid acc= 0.8475, valid loss= 0.7520
Epoch 7: train acc= 0.8439, train loss= 0.7553 | valid acc= 0.8485, valid loss= 0.7500
Epoch 8: train acc= 0.8446, train loss= 0.7537 | valid acc= 0.8485, valid loss= 0.7486
Epoch 9: train acc= 0.8449, train loss= 0.7525 | valid acc= 0.8500, valid loss= 0.7476
Epoch 10: train acc= 0.8451, train loss= 0.7516 | valid acc= 0.8502, valid loss= 0.7469
	valid acc = 0.8501666666666666, valid loss = 0.7468579011961676

This is for model n1= 40, n2 = 20, activation function = relu
Epoch 1: train acc= 0.8256, train loss= 0.8910 | valid acc= 0.8317, valid loss= 0.8841
Epoch 2: train acc= 0.8363, train loss= 0.7657 | valid acc= 0.8443, valid loss= 0.7600
Epoch 3: train acc= 0.8403, train loss= 0.7246 | valid acc= 0.8475, valid loss= 0.7197
Epoch 4: train acc= 0.8421, train loss= 0.7094 | valid acc= 0.8495, valid loss= 0.7051
Epoch 5: train acc= 0.8429, train loss= 0.7031 | valid acc= 0.8488, valid loss= 0.6995
Epoch 6: train acc= 0.8443, train loss= 0.6994 | valid acc= 0.8510, valid loss= 0.6962
Epoch 7: train acc= 0.8448, train loss= 0.6971 | valid acc= 0.8517, valid loss= 0.6942
Epoch 8: train acc= 0.8452, train loss= 0.6961 | valid acc= 0.8507, valid loss= 0.6935
Epoch 9: train acc= 0.8454, train loss= 0.6949 | valid acc= 0.8512, valid loss= 0.6924
Epoch 10: train acc= 0.8458, train loss= 0.6937 | valid acc= 0.8520, valid loss= 0.6914
	valid acc = 0.852, valid loss = 0.6914116841091157

This is for model n1= 40, n2 = 40, activation function = sigmoid
Epoch 1: train acc= 0.5500, train loss= 1.8929 | valid acc= 0.5500, valid loss= 1.8891
Epoch 2: train acc= 0.5796, train loss= 1.6515 | valid acc= 0.5815, valid loss= 1.6473
Epoch 3: train acc= 0.5964, train loss= 1.5792 | valid acc= 0.5997, valid loss= 1.5746
Epoch 4: train acc= 0.6082, train loss= 1.5494 | valid acc= 0.6098, valid loss= 1.5446
Epoch 5: train acc= 0.6198, train loss= 1.5339 | valid acc= 0.6212, valid loss= 1.5289
Epoch 6: train acc= 0.6316, train loss= 1.5238 | valid acc= 0.6360, valid loss= 1.5188
Epoch 7: train acc= 0.6455, train loss= 1.5157 | valid acc= 0.6468, valid loss= 1.5107
Epoch 8: train acc= 0.6604, train loss= 1.5071 | valid acc= 0.6618, valid loss= 1.5024
Epoch 9: train acc= 0.6738, train loss= 1.4965 | valid acc= 0.6722, valid loss= 1.4925
Epoch 10: train acc= 0.6830, train loss= 1.4858 | valid acc= 0.6805, valid loss= 1.4824
	valid acc = 0.6805, valid loss = 1.4823851180197463

This is for model n1= 40, n2 = 40, activation function = tanh
Epoch 1: train acc= 0.8302, train loss= 0.9330 | valid acc= 0.8393, valid loss= 0.9244
Epoch 2: train acc= 0.8391, train loss= 0.8040 | valid acc= 0.8458, valid loss= 0.7967
Epoch 3: train acc= 0.8428, train loss= 0.7601 | valid acc= 0.8472, valid loss= 0.7535
Epoch 4: train acc= 0.8444, train loss= 0.7431 | valid acc= 0.8487, valid loss= 0.7371
Epoch 5: train acc= 0.8456, train loss= 0.7355 | valid acc= 0.8493, valid loss= 0.7299
Epoch 6: train acc= 0.8462, train loss= 0.7313 | valid acc= 0.8503, valid loss= 0.7259
Epoch 7: train acc= 0.8467, train loss= 0.7286 | valid acc= 0.8508, valid loss= 0.7235
Epoch 8: train acc= 0.8469, train loss= 0.7267 | valid acc= 0.8513, valid loss= 0.7217
Epoch 9: train acc= 0.8475, train loss= 0.7251 | valid acc= 0.8522, valid loss= 0.7203
Epoch 10: train acc= 0.8478, train loss= 0.7237 | valid acc= 0.8525, valid loss= 0.7190
	valid acc = 0.8525, valid loss = 0.7190313472134627

This is for model n1= 40, n2 = 40, activation function = relu
Epoch 1: train acc= 0.8209, train loss= 0.9281 | valid acc= 0.8328, valid loss= 0.9171
Epoch 2: train acc= 0.8326, train loss= 0.7785 | valid acc= 0.8440, valid loss= 0.7703
Epoch 3: train acc= 0.8372, train loss= 0.7293 | valid acc= 0.8467, valid loss= 0.7229
Epoch 4: train acc= 0.8393, train loss= 0.7111 | valid acc= 0.8477, valid loss= 0.7055
Epoch 5: train acc= 0.8405, train loss= 0.7035 | valid acc= 0.8482, valid loss= 0.6980
Epoch 6: train acc= 0.8413, train loss= 0.6995 | valid acc= 0.8488, valid loss= 0.6945
Epoch 7: train acc= 0.8419, train loss= 0.6969 | valid acc= 0.8487, valid loss= 0.6922
Epoch 8: train acc= 0.8432, train loss= 0.6946 | valid acc= 0.8490, valid loss= 0.6902
Epoch 9: train acc= 0.8437, train loss= 0.6931 | valid acc= 0.8497, valid loss= 0.6888
Epoch 10: train acc= 0.8443, train loss= 0.6920 | valid acc= 0.8498, valid loss= 0.6879
	valid acc = 0.8498333333333333, valid loss = 0.6879278224979427

The best model is with n1= 40, n2 = 40, activation function = tanh


best_model.model.save('models/best_cnn.h5')

WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.


class SAM(tf.keras.optimizers.Optimizer):
    def __init__(self, base_optimizer, rho=0.05):
        super(SAM, self).__init__(name="SAM")
        self.base_optimizer = base_optimizer
        self.rho = 0.05
        self.epsilon = 1e-12
    
    def apply_gradients (self,zipped_grads_and_trainable_variables):
        # unzip the values taken in from the model
        grads, trainable = zip(*zipped_grads_and_trainable_variables)
        
        #zip it back and pass it into our base optimizer (Adam in this case)
        self.base_optimizer.apply_gradients(zip(grads,trainable))
        
        #initialize sharpness value
        sharpness = 0.0
        #compute the total sharpness value by taking the square of each gradients and reduce it into a single value
        for grad in grads:
            sharpness += tf.reduce_sum(tf.square(grad))
        
        #initalize scaled_gradients array
        scaled_grad = []
        #apply sharpness to each gradient value
        for grad in grads:
            scaled_grad.append(self.rho * grad / (sharpness + self.epsilon))
        
        #reapply gradients with the base optimizer with the sharpness adjusted values
        self.base_optimizer.apply_gradients(zip(scaled_grad,trainable))
        
sam = SAM(opt)
opt = tf.keras.optimizers.Adam()
dnn = DNN(40,40,'tanh',optimizer=sam, epochs=50, batch_size=32)
dnn.build()
dnn.train()
dnn.evaluate_test_set()

Epoch 1: train acc= 0.8462, train loss= 0.5069 | valid acc= 0.8380, valid loss= 0.5244
Epoch 2: train acc= 0.8607, train loss= 0.4787 | valid acc= 0.8503, valid loss= 0.5011
Epoch 3: train acc= 0.8665, train loss= 0.4675 | valid acc= 0.8552, valid loss= 0.4931
Epoch 4: train acc= 0.8694, train loss= 0.4604 | valid acc= 0.8555, valid loss= 0.4882
Epoch 5: train acc= 0.8709, train loss= 0.4574 | valid acc= 0.8582, valid loss= 0.4876
Epoch 6: train acc= 0.8725, train loss= 0.4542 | valid acc= 0.8590, valid loss= 0.4862
Epoch 7: train acc= 0.8724, train loss= 0.4551 | valid acc= 0.8582, valid loss= 0.4887
Epoch 8: train acc= 0.8725, train loss= 0.4551 | valid acc= 0.8582, valid loss= 0.4901
Epoch 9: train acc= 0.8746, train loss= 0.4503 | valid acc= 0.8605, valid loss= 0.4855
Epoch 10: train acc= 0.8751, train loss= 0.4510 | valid acc= 0.8605, valid loss= 0.4865
Epoch 11: train acc= 0.8760, train loss= 0.4496 | valid acc= 0.8617, valid loss= 0.4848
Epoch 12: train acc= 0.8762, train loss= 0.4482 | valid acc= 0.8630, valid loss= 0.4836
Epoch 13: train acc= 0.8771, train loss= 0.4483 | valid acc= 0.8628, valid loss= 0.4825
Epoch 14: train acc= 0.8758, train loss= 0.4510 | valid acc= 0.8597, valid loss= 0.4860
Epoch 15: train acc= 0.8776, train loss= 0.4484 | valid acc= 0.8617, valid loss= 0.4842
Epoch 16: train acc= 0.8778, train loss= 0.4465 | valid acc= 0.8623, valid loss= 0.4826
Epoch 17: train acc= 0.8796, train loss= 0.4409 | valid acc= 0.8623, valid loss= 0.4777
Epoch 18: train acc= 0.8803, train loss= 0.4405 | valid acc= 0.8638, valid loss= 0.4774
Epoch 19: train acc= 0.8801, train loss= 0.4418 | valid acc= 0.8633, valid loss= 0.4792
Epoch 20: train acc= 0.8805, train loss= 0.4405 | valid acc= 0.8637, valid loss= 0.4780
Epoch 21: train acc= 0.8815, train loss= 0.4393 | valid acc= 0.8628, valid loss= 0.4775
Epoch 22: train acc= 0.8814, train loss= 0.4401 | valid acc= 0.8648, valid loss= 0.4783
Epoch 23: train acc= 0.8818, train loss= 0.4391 | valid acc= 0.8640, valid loss= 0.4774
Epoch 24: train acc= 0.8816, train loss= 0.4390 | valid acc= 0.8653, valid loss= 0.4779
Epoch 25: train acc= 0.8820, train loss= 0.4383 | valid acc= 0.8655, valid loss= 0.4771
Epoch 26: train acc= 0.8817, train loss= 0.4373 | valid acc= 0.8660, valid loss= 0.4765
Epoch 27: train acc= 0.8823, train loss= 0.4361 | valid acc= 0.8653, valid loss= 0.4758
Epoch 28: train acc= 0.8823, train loss= 0.4363 | valid acc= 0.8662, valid loss= 0.4758
Epoch 29: train acc= 0.8833, train loss= 0.4341 | valid acc= 0.8665, valid loss= 0.4742
Epoch 30: train acc= 0.8826, train loss= 0.4358 | valid acc= 0.8657, valid loss= 0.4762
Epoch 31: train acc= 0.8832, train loss= 0.4335 | valid acc= 0.8668, valid loss= 0.4739
Epoch 32: train acc= 0.8827, train loss= 0.4345 | valid acc= 0.8665, valid loss= 0.4757
Epoch 33: train acc= 0.8823, train loss= 0.4349 | valid acc= 0.8658, valid loss= 0.4762
Epoch 34: train acc= 0.8841, train loss= 0.4320 | valid acc= 0.8683, valid loss= 0.4740
Epoch 35: train acc= 0.8850, train loss= 0.4310 | valid acc= 0.8665, valid loss= 0.4732
Epoch 36: train acc= 0.8837, train loss= 0.4327 | valid acc= 0.8667, valid loss= 0.4753
Epoch 37: train acc= 0.8845, train loss= 0.4306 | valid acc= 0.8682, valid loss= 0.4734
Epoch 38: train acc= 0.8836, train loss= 0.4328 | valid acc= 0.8668, valid loss= 0.4755
Epoch 39: train acc= 0.8843, train loss= 0.4307 | valid acc= 0.8675, valid loss= 0.4735
Epoch 40: train acc= 0.8848, train loss= 0.4294 | valid acc= 0.8660, valid loss= 0.4718
Epoch 41: train acc= 0.8846, train loss= 0.4307 | valid acc= 0.8650, valid loss= 0.4735
Epoch 42: train acc= 0.8836, train loss= 0.4318 | valid acc= 0.8658, valid loss= 0.4750
Epoch 43: train acc= 0.8839, train loss= 0.4317 | valid acc= 0.8653, valid loss= 0.4754
Epoch 44: train acc= 0.8841, train loss= 0.4312 | valid acc= 0.8657, valid loss= 0.4748
Epoch 45: train acc= 0.8832, train loss= 0.4332 | valid acc= 0.8638, valid loss= 0.4768
Epoch 46: train acc= 0.8835, train loss= 0.4337 | valid acc= 0.8630, valid loss= 0.4770
Epoch 47: train acc= 0.8835, train loss= 0.4315 | valid acc= 0.8652, valid loss= 0.4751
Epoch 48: train acc= 0.8836, train loss= 0.4318 | valid acc= 0.8652, valid loss= 0.4751
Epoch 49: train acc= 0.8838, train loss= 0.4322 | valid acc= 0.8653, valid loss= 0.4757
Epoch 50: train acc= 0.8829, train loss= 0.4339 | valid acc= 0.8647, valid loss= 0.4777

'test acc = 0.8616, test loss = 0.4981'


%load_ext autoreload
%autoreload 2


! pip install imutils

Requirement already satisfied: imutils in c:\users\manut\anaconda3\envs\gpu\lib\site-packages (0.5.4)


import os
import matplotlib.pyplot as plt
plt.style.use('ggplot')
%matplotlib inline
import models
from models import SimplePreprocessor, DatasetManager, DefaultModel


def create_label_folder_dict(adir):
    sub_folders= [folder for folder in os.listdir(adir)
                  if os.path.isdir(os.path.join(adir, folder))]
    label_folder_dict= dict()
    for folder in sub_folders:
        item= {folder: os.path.abspath(os.path.join(adir, folder))}
        label_folder_dict.update(item)
    return label_folder_dict


label_folder_dict= create_label_folder_dict("./datasets/FIT5215_Dataset")


sp = SimplePreprocessor(width=32, height=32)
data_manager = DatasetManager([sp])
data_manager.load(label_folder_dict, verbose=100)
data_manager.process_data_label()
data_manager.train_valid_test_split()

birds 512
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
bottles 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
breads 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
butterfiles 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cakes 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
cats 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
chickens 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cows 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
dogs 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
ducks 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
elephants 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
fishes 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
handguns 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
horses 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lions 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lipsticks 400
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
seals 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
snakes 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
spiders 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
vases 368
Processed 100/500
Processed 200/500
Processed 300/500


print(data_manager.X_train.shape, data_manager.y_train.shape)
print(data_manager.X_valid.shape, data_manager.y_valid.shape)
print(data_manager.X_test.shape, data_manager.y_test.shape)
print(data_manager.classes)

(7560, 32, 32, 3) (7560,)
(946, 32, 32, 3) (946,)
(946, 32, 32, 3) (946,)
['birds' 'bottles' 'breads' 'butterfiles' 'cakes' 'cats' 'chickens' 'cows'
 'dogs' 'ducks' 'elephants' 'fishes' 'handguns' 'horses' 'lions'
 'lipsticks' 'seals' 'snakes' 'spiders' 'vases']


network1 = DefaultModel(name='network1',
                       num_classes=len(data_manager.classes),
                       optimizer='sgd',
                       batch_size= 128,
                       num_epochs = 20,
                       learning_rate=0.1)


network1.build_cnn()
network1.summary()

Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d (Conv2D)             (None, 32, 32, 32)        896       
                                                                 
 conv2d_1 (Conv2D)           (None, 32, 32, 32)        9248      
                                                                 
 average_pooling2d (AverageP  (None, 16, 16, 32)       0         
 ooling2D)                                                       
                                                                 
 conv2d_2 (Conv2D)           (None, 16, 16, 64)        18496     
                                                                 
 conv2d_3 (Conv2D)           (None, 16, 16, 64)        36928     
                                                                 
 average_pooling2d_1 (Averag  (None, 8, 8, 64)         0         
 ePooling2D)                                                     
                                                                 
 flatten (Flatten)           (None, 4096)              0         
                                                                 
 dense_3 (Dense)             (None, 20)                81940     
                                                                 
=================================================================
Total params: 147,508
Trainable params: 147,508
Non-trainable params: 0
_________________________________________________________________
None


network1.fit(data_manager, batch_size = 64, num_epochs = 20)

Epoch 1/20
119/119 [==============================] - 10s 42ms/step - loss: 2.8829 - accuracy: 0.1139 - val_loss: 2.9179 - val_accuracy: 0.1226
Epoch 2/20
119/119 [==============================] - 5s 40ms/step - loss: 2.5422 - accuracy: 0.2311 - val_loss: 2.8218 - val_accuracy: 0.1797
Epoch 3/20
119/119 [==============================] - 5s 39ms/step - loss: 2.3153 - accuracy: 0.2954 - val_loss: 3.2079 - val_accuracy: 0.1723
Epoch 4/20
119/119 [==============================] - 5s 39ms/step - loss: 2.2063 - accuracy: 0.3272 - val_loss: 3.3495 - val_accuracy: 0.1734
Epoch 5/20
119/119 [==============================] - 5s 40ms/step - loss: 2.0907 - accuracy: 0.3567 - val_loss: 2.5826 - val_accuracy: 0.2093
Epoch 6/20
119/119 [==============================] - 5s 40ms/step - loss: 1.9664 - accuracy: 0.3976 - val_loss: 2.9541 - val_accuracy: 0.2178
Epoch 7/20
119/119 [==============================] - 5s 40ms/step - loss: 1.8645 - accuracy: 0.4266 - val_loss: 2.5988 - val_accuracy: 0.2347
Epoch 8/20
119/119 [==============================] - 5s 40ms/step - loss: 1.7803 - accuracy: 0.4540 - val_loss: 2.3764 - val_accuracy: 0.2801
Epoch 9/20
119/119 [==============================] - 5s 40ms/step - loss: 1.6480 - accuracy: 0.4926 - val_loss: 6.8173 - val_accuracy: 0.1564
Epoch 10/20
119/119 [==============================] - 5s 40ms/step - loss: 1.7674 - accuracy: 0.4706 - val_loss: 2.4231 - val_accuracy: 0.3277
Epoch 11/20
119/119 [==============================] - 5s 41ms/step - loss: 1.4942 - accuracy: 0.5414 - val_loss: 2.8209 - val_accuracy: 0.2928
Epoch 12/20
119/119 [==============================] - 5s 41ms/step - loss: 1.3854 - accuracy: 0.5694 - val_loss: 2.6143 - val_accuracy: 0.3510
Epoch 13/20
119/119 [==============================] - 5s 40ms/step - loss: 1.2694 - accuracy: 0.6041 - val_loss: 2.4816 - val_accuracy: 0.3552
Epoch 14/20
119/119 [==============================] - 5s 41ms/step - loss: 1.1388 - accuracy: 0.6496 - val_loss: 2.6341 - val_accuracy: 0.3510
Epoch 15/20
119/119 [==============================] - 5s 41ms/step - loss: 1.0294 - accuracy: 0.6776 - val_loss: 2.6812 - val_accuracy: 0.3721
Epoch 16/20
119/119 [==============================] - 5s 41ms/step - loss: 0.9249 - accuracy: 0.7063 - val_loss: 2.9345 - val_accuracy: 0.3689
Epoch 17/20
119/119 [==============================] - 5s 42ms/step - loss: 0.8156 - accuracy: 0.7398 - val_loss: 3.0814 - val_accuracy: 0.3615
Epoch 18/20
119/119 [==============================] - 5s 43ms/step - loss: 0.7208 - accuracy: 0.7713 - val_loss: 3.7819 - val_accuracy: 0.3319
Epoch 19/20
119/119 [==============================] - 5s 42ms/step - loss: 0.6296 - accuracy: 0.8017 - val_loss: 4.1202 - val_accuracy: 0.3214
Epoch 20/20
119/119 [==============================] - 5s 42ms/step - loss: 0.5418 - accuracy: 0.8250 - val_loss: 4.7613 - val_accuracy: 0.3129


network1.compute_accuracy(data_manager.X_test, data_manager.y_test)

15/15 [==============================] - 0s 13ms/step - loss: 4.8948 - accuracy: 0.3055

0.3054968287526427


network1.plot_progress()


network1.predict(data_manager.X_test[0:10])

1/1 [==============================] - 0s 103ms/step

array([ 2, 11,  9, 18, 15,  9,  5,  0,  9, 10], dtype=int64)


network1.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

30/30 [==============================] - 0s 7ms/step

<Figure size 640x480 with 0 Axes>


import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, models
from tensorflow.keras.layers import GlobalAveragePooling2D
import numpy as np
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping


tf.random.set_seed(1234)


# model class adapted from tutorials 
class YourModel(DefaultModel):
    def __init__(self,num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose,
                 name='network1',
                 width=32, height=32, depth=3,
                 num_classes=20, 
                 is_augmentation = False,
                 activation_func='relu',
                 optimizer='adam',
                 batch_size=128,
                 num_epochs= 30):
        super(YourModel, self).__init__(name, width, height, depth, num_classes, is_augmentation, 
                                        activation_func, optimizer, batch_size, num_epochs, 
                                        learning_rate, verbose)
        self.num_channels = num_channels
        self.mean_pool = mean_pool
        self.batch_norm = batch_norm
        self.use_skip = use_skip
        self.blocks = blocks
    
    def build_cnn(self,x):
        # Resblock code for builing each block
        # x1 takes the inputs of x and passes it through a conv layer
        x1 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x)
        # if batch_norm is true, pass x1 through a batch norm layer
        if self.batch_norm:
            x1 = layers.BatchNormalization() (x1)
        # apply activation function to x1
        x1 = layers.Activation('relu') (x1)
        # pass values of x1 into a conv layer
        x2 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x1)
        #if value of x1 is
        if self.batch_norm:
                x2 = layers.BatchNormalization()(x2)
        if x.shape != x2.shape:
            if x2.shape[3] > x.shape[3]:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x2.shape[3] - x.shape[3] , 0]])
                x = tf.pad(x, pad_tns, mode ='CONSTANT', constant_values=0)
            else:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x.shape[3] - x2.shape[3] , 0]])
                x = tf.pad(x2, pad_tns, mode ='CONSTANT', constant_values=0)
        x2_skip = layers.add([x, x2])
        x2_skip = layers.Activation('relu')(x2_skip)
        if self.mean_pool:
            output_layer = layers.AveragePooling2D(pool_size=(2, 2), padding='same')(x2_skip)
        else: 
            output_layer = x2_skip
        return output_layer
    
    def build_resnet(self):
        self.input_layer = layers.Input(shape=(self.width, self.height, self.depth))
        x = self.input_layer
        for i in range (self.blocks):
            x = self.build_cnn(x)
            self.num_channels = self.num_channels*2
            
        output_layer = GlobalAveragePooling2D()(x)
        output_layer = layers.Flatten()(output_layer)
        output_layer = layers.Dense(self.num_classes, activation='softmax')(output_layer)
        self.model = tf.keras.models.Model(inputs=self.input_layer, outputs=output_layer)
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])


#Your run here
# num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose
testModel = YourModel(16,2,True,True,True,0.001,True)
testModel.build_resnet()
testModel.summary()
testModel.fit(data_manager, batch_size = 16, num_epochs = 20)

Model: "model"
__________________________________________________________________________________________________
 Layer (type)                   Output Shape         Param #     Connected to                     
==================================================================================================
 input_1 (InputLayer)           [(None, 32, 32, 3)]  0           []                               
                                                                                                  
 conv2d_4 (Conv2D)              (None, 32, 32, 32)   896         ['input_1[0][0]']                
                                                                                                  
 batch_normalization (BatchNorm  (None, 32, 32, 32)  128         ['conv2d_4[0][0]']               
 alization)                                                                                       
                                                                                                  
 activation (Activation)        (None, 32, 32, 32)   0           ['batch_normalization[0][0]']    
                                                                                                  
 conv2d_5 (Conv2D)              (None, 32, 32, 32)   9248        ['activation[0][0]']             
                                                                                                  
 tf.compat.v1.pad (TFOpLambda)  (None, 32, 32, 32)   0           ['input_1[0][0]']                
                                                                                                  
 batch_normalization_1 (BatchNo  (None, 32, 32, 32)  128         ['conv2d_5[0][0]']               
 rmalization)                                                                                     
                                                                                                  
 add (Add)                      (None, 32, 32, 32)   0           ['tf.compat.v1.pad[0][0]',       
                                                                  'batch_normalization_1[0][0]']  
                                                                                                  
 activation_1 (Activation)      (None, 32, 32, 32)   0           ['add[0][0]']                    
                                                                                                  
 average_pooling2d_2 (AveragePo  (None, 16, 16, 32)  0           ['activation_1[0][0]']           
 oling2D)                                                                                         
                                                                                                  
 conv2d_6 (Conv2D)              (None, 16, 16, 64)   18496       ['average_pooling2d_2[0][0]']    
                                                                                                  
 batch_normalization_2 (BatchNo  (None, 16, 16, 64)  256         ['conv2d_6[0][0]']               
 rmalization)                                                                                     
                                                                                                  
 activation_2 (Activation)      (None, 16, 16, 64)   0           ['batch_normalization_2[0][0]']  
                                                                                                  
 conv2d_7 (Conv2D)              (None, 16, 16, 64)   36928       ['activation_2[0][0]']           
                                                                                                  
 tf.compat.v1.pad_1 (TFOpLambda  (None, 16, 16, 64)  0           ['average_pooling2d_2[0][0]']    
 )                                                                                                
                                                                                                  
 batch_normalization_3 (BatchNo  (None, 16, 16, 64)  256         ['conv2d_7[0][0]']               
 rmalization)                                                                                     
                                                                                                  
 add_1 (Add)                    (None, 16, 16, 64)   0           ['tf.compat.v1.pad_1[0][0]',     
                                                                  'batch_normalization_3[0][0]']  
                                                                                                  
 activation_3 (Activation)      (None, 16, 16, 64)   0           ['add_1[0][0]']                  
                                                                                                  
 average_pooling2d_3 (AveragePo  (None, 8, 8, 64)    0           ['activation_3[0][0]']           
 oling2D)                                                                                         
                                                                                                  
 global_average_pooling2d (Glob  (None, 64)          0           ['average_pooling2d_3[0][0]']    
 alAveragePooling2D)                                                                              
                                                                                                  
 flatten_1 (Flatten)            (None, 64)           0           ['global_average_pooling2d[0][0]'
                                                                 ]                                
                                                                                                  
 dense_4 (Dense)                (None, 20)           1300        ['flatten_1[0][0]']              
                                                                                                  
==================================================================================================
Total params: 67,636
Trainable params: 67,252
Non-trainable params: 384
__________________________________________________________________________________________________
None
Epoch 1/20
473/473 [==============================] - 9s 16ms/step - loss: 2.5454 - accuracy: 0.2274 - val_loss: 2.4879 - val_accuracy: 0.2357
Epoch 2/20
473/473 [==============================] - 7s 15ms/step - loss: 2.2405 - accuracy: 0.3165 - val_loss: 2.2155 - val_accuracy: 0.3087
Epoch 3/20
473/473 [==============================] - 7s 15ms/step - loss: 2.0808 - accuracy: 0.3583 - val_loss: 2.4101 - val_accuracy: 0.2558
Epoch 4/20
473/473 [==============================] - 7s 15ms/step - loss: 1.9518 - accuracy: 0.3942 - val_loss: 1.9236 - val_accuracy: 0.3932
Epoch 5/20
473/473 [==============================] - 7s 15ms/step - loss: 1.8321 - accuracy: 0.4283 - val_loss: 1.7975 - val_accuracy: 0.4197
Epoch 6/20
473/473 [==============================] - 7s 15ms/step - loss: 1.7244 - accuracy: 0.4655 - val_loss: 1.7212 - val_accuracy: 0.4641
Epoch 7/20
473/473 [==============================] - 7s 16ms/step - loss: 1.6426 - accuracy: 0.4907 - val_loss: 1.8694 - val_accuracy: 0.4123
Epoch 8/20
473/473 [==============================] - 8s 16ms/step - loss: 1.5542 - accuracy: 0.5144 - val_loss: 1.5336 - val_accuracy: 0.5402
Epoch 9/20
473/473 [==============================] - 8s 16ms/step - loss: 1.4966 - accuracy: 0.5282 - val_loss: 1.7946 - val_accuracy: 0.4450
Epoch 10/20
473/473 [==============================] - 7s 15ms/step - loss: 1.4419 - accuracy: 0.5516 - val_loss: 2.1555 - val_accuracy: 0.3552
Epoch 11/20
473/473 [==============================] - 7s 15ms/step - loss: 1.3794 - accuracy: 0.5698 - val_loss: 1.5656 - val_accuracy: 0.5116
Epoch 12/20
473/473 [==============================] - 7s 15ms/step - loss: 1.3423 - accuracy: 0.5742 - val_loss: 1.4894 - val_accuracy: 0.5349
Epoch 13/20
473/473 [==============================] - 7s 15ms/step - loss: 1.2964 - accuracy: 0.5955 - val_loss: 1.5869 - val_accuracy: 0.4979
Epoch 14/20
473/473 [==============================] - 7s 16ms/step - loss: 1.2537 - accuracy: 0.6050 - val_loss: 1.4339 - val_accuracy: 0.5507
Epoch 15/20
473/473 [==============================] - 7s 16ms/step - loss: 1.2226 - accuracy: 0.6147 - val_loss: 1.4221 - val_accuracy: 0.5412
Epoch 16/20
473/473 [==============================] - 7s 16ms/step - loss: 1.1923 - accuracy: 0.6282 - val_loss: 1.4886 - val_accuracy: 0.5370
Epoch 17/20
473/473 [==============================] - 7s 16ms/step - loss: 1.1622 - accuracy: 0.6349 - val_loss: 1.3364 - val_accuracy: 0.5920
Epoch 18/20
473/473 [==============================] - 8s 16ms/step - loss: 1.1086 - accuracy: 0.6495 - val_loss: 1.3345 - val_accuracy: 0.5867
Epoch 19/20
473/473 [==============================] - 8s 16ms/step - loss: 1.0759 - accuracy: 0.6632 - val_loss: 1.3187 - val_accuracy: 0.5888
Epoch 20/20
473/473 [==============================] - 8s 16ms/step - loss: 1.0528 - accuracy: 0.6675 - val_loss: 1.4775 - val_accuracy: 0.5772


testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)
testModel.plot_progress()
testModel.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

15/15 [==============================] - 0s 17ms/step - loss: 1.5110 - accuracy: 0.5560

30/30 [==============================] - 0s 7ms/step

<Figure size 640x480 with 0 Axes>


#Save baseline model
testModel.model.save('models/base_dnn.h5')


#Insert your code here. You can add more cells if necessary
num_blocks = [3,4]
use_skip = [True,False]
mean_pool = [True,False]
learning_rate = [0.001,0.0001]
best_accuracy = -float('inf')
bestModel = None

# num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose

for blocks in num_blocks:
    for skip in use_skip:
        for pool in mean_pool:
            for rate in learning_rate:
                testModel = YourModel(16,blocks,pool,True,skip,rate,True)
                testModel.build_resnet()
                testModel.fit(data_manager, batch_size = 64, num_epochs = 30)
                acc = testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)
                if acc > best_accuracy:
                    best_model = testModel
                    best_accuracy = acc
                    best_config = (blocks,skip,pool,rate,acc)
                    
best_model.model.save('models/best_base_dnn_config.h5')

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.

(None, 32, 32, 3) (None, 32, 32, 16)
(None, 32, 32, 16)
(None, 16, 16, 16) (None, 16, 16, 32)
(None, 16, 16, 32)
(None, 8, 8, 32) (None, 8, 8, 64)
(None, 8, 8, 64)
Epoch 1/30
119/119 [==============================] - 13s 105ms/step - loss: 2.4923 - accuracy: 0.2440 - val_loss: 3.0471 - val_accuracy: 0.1025
Epoch 2/30
119/119 [==============================] - 12s 101ms/step - loss: 2.1277 - accuracy: 0.3467 - val_loss: 3.2528 - val_accuracy: 0.1279
Epoch 3/30
119/119 [==============================] - 12s 98ms/step - loss: 1.9461 - accuracy: 0.4070 - val_loss: 2.8097 - val_accuracy: 0.1977
Epoch 4/30
119/119 [==============================] - 12s 99ms/step - loss: 1.8313 - accuracy: 0.4411 - val_loss: 2.1317 - val_accuracy: 0.3414
Epoch 5/30
119/119 [==============================] - 12s 100ms/step - loss: 1.7085 - accuracy: 0.4763 - val_loss: 1.9802 - val_accuracy: 0.4038
Epoch 6/30
119/119 [==============================] - 12s 101ms/step - loss: 1.6147 - accuracy: 0.5029 - val_loss: 1.8196 - val_accuracy: 0.4366
Epoch 7/30
119/119 [==============================] - 12s 99ms/step - loss: 1.5251 - accuracy: 0.5347 - val_loss: 1.9529 - val_accuracy: 0.4186
Epoch 8/30
119/119 [==============================] - 12s 102ms/step - loss: 1.4721 - accuracy: 0.5446 - val_loss: 1.8760 - val_accuracy: 0.4207
Epoch 9/30
119/119 [==============================] - 12s 101ms/step - loss: 1.3922 - accuracy: 0.5731 - val_loss: 1.7467 - val_accuracy: 0.4609
Epoch 10/30
119/119 [==============================] - 12s 99ms/step - loss: 1.3194 - accuracy: 0.5939 - val_loss: 1.6478 - val_accuracy: 0.4799
Epoch 11/30
119/119 [==============================] - 13s 108ms/step - loss: 1.2437 - accuracy: 0.6148 - val_loss: 1.7037 - val_accuracy: 0.4820
Epoch 12/30
119/119 [==============================] - 12s 103ms/step - loss: 1.2083 - accuracy: 0.6239 - val_loss: 1.7971 - val_accuracy: 0.4609
Epoch 13/30
119/119 [==============================] - 13s 111ms/step - loss: 1.1413 - accuracy: 0.6431 - val_loss: 1.6978 - val_accuracy: 0.4736
Epoch 14/30
119/119 [==============================] - 12s 102ms/step - loss: 1.1088 - accuracy: 0.6541 - val_loss: 1.7140 - val_accuracy: 0.4704
Epoch 15/30
119/119 [==============================] - 12s 100ms/step - loss: 1.0597 - accuracy: 0.6724 - val_loss: 1.7007 - val_accuracy: 0.5053
Epoch 16/30
119/119 [==============================] - 12s 100ms/step - loss: 1.0065 - accuracy: 0.6817 - val_loss: 2.1691 - val_accuracy: 0.3827
Epoch 17/30
119/119 [==============================] - 12s 101ms/step - loss: 0.9765 - accuracy: 0.6948 - val_loss: 1.6123 - val_accuracy: 0.5275
Epoch 18/30
119/119 [==============================] - 12s 102ms/step - loss: 0.9267 - accuracy: 0.7132 - val_loss: 1.5606 - val_accuracy: 0.5254
Epoch 19/30
119/119 [==============================] - 12s 102ms/step - loss: 0.8760 - accuracy: 0.7284 - val_loss: 1.6391 - val_accuracy: 0.4884
Epoch 20/30
119/119 [==============================] - 12s 101ms/step - loss: 0.8644 - accuracy: 0.7283 - val_loss: 1.6272 - val_accuracy: 0.5328
Epoch 21/30
119/119 [==============================] - 12s 102ms/step - loss: 0.8107 - accuracy: 0.7475 - val_loss: 1.7318 - val_accuracy: 0.5106
Epoch 22/30
119/119 [==============================] - 12s 101ms/step - loss: 0.7856 - accuracy: 0.7574 - val_loss: 1.6059 - val_accuracy: 0.4968
Epoch 23/30
119/119 [==============================] - 12s 102ms/step - loss: 0.7273 - accuracy: 0.7800 - val_loss: 1.6060 - val_accuracy: 0.5391
Epoch 24/30
119/119 [==============================] - 12s 101ms/step - loss: 0.6959 - accuracy: 0.7876 - val_loss: 1.9895 - val_accuracy: 0.4683
Epoch 25/30
119/119 [==============================] - 12s 102ms/step - loss: 0.6657 - accuracy: 0.7954 - val_loss: 2.0533 - val_accuracy: 0.4545
Epoch 26/30
119/119 [==============================] - 12s 104ms/step - loss: 0.6310 - accuracy: 0.8049 - val_loss: 1.6096 - val_accuracy: 0.5307
Epoch 27/30
119/119 [==============================] - 12s 102ms/step - loss: 0.6205 - accuracy: 0.8063 - val_loss: 1.7208 - val_accuracy: 0.5021
Epoch 28/30
119/119 [==============================] - 13s 106ms/step - loss: 0.5879 - accuracy: 0.8222 - val_loss: 1.6549 - val_accuracy: 0.5359
Epoch 29/30
119/119 [==============================] - 15s 127ms/step - loss: 0.5777 - accuracy: 0.8200 - val_loss: 1.7144 - val_accuracy: 0.5539
Epoch 30/30
119/119 [==============================] - 12s 102ms/step - loss: 0.5184 - accuracy: 0.8430 - val_loss: 1.7804 - val_accuracy: 0.5518
15/15 [==============================] - 1s 35ms/step - loss: 1.6913 - accuracy: 0.5465

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.

(None, 32, 32, 3) (None, 32, 32, 16)
(None, 32, 32, 16)
(None, 16, 16, 16) (None, 16, 16, 32)
(None, 16, 16, 32)

WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.

(None, 8, 8, 32) (None, 8, 8, 64)
(None, 8, 8, 64)
Epoch 1/30
119/119 [==============================] - 15s 124ms/step - loss: 2.9280 - accuracy: 0.1108 - val_loss: 2.9893 - val_accuracy: 0.0772
Epoch 2/30
119/119 [==============================] - 14s 114ms/step - loss: 2.6646 - accuracy: 0.2005 - val_loss: 2.9263 - val_accuracy: 0.0899
Epoch 3/30
119/119 [==============================] - 12s 103ms/step - loss: 2.5306 - accuracy: 0.2574 - val_loss: 2.6735 - val_accuracy: 0.1660
Epoch 4/30
119/119 [==============================] - 12s 101ms/step - loss: 2.4293 - accuracy: 0.2880 - val_loss: 2.4811 - val_accuracy: 0.2273
Epoch 5/30
119/119 [==============================] - 12s 100ms/step - loss: 2.3456 - accuracy: 0.3094 - val_loss: 2.3525 - val_accuracy: 0.2738
Epoch 6/30
119/119 [==============================] - 12s 100ms/step - loss: 2.2772 - accuracy: 0.3299 - val_loss: 2.2804 - val_accuracy: 0.2907
Epoch 7/30
119/119 [==============================] - 12s 100ms/step - loss: 2.2209 - accuracy: 0.3380 - val_loss: 2.2241 - val_accuracy: 0.3044
Epoch 8/30
119/119 [==============================] - 12s 102ms/step - loss: 2.1652 - accuracy: 0.3552 - val_loss: 2.2030 - val_accuracy: 0.3034
Epoch 9/30
119/119 [==============================] - 13s 108ms/step - loss: 2.1184 - accuracy: 0.3657 - val_loss: 2.1242 - val_accuracy: 0.3520
Epoch 10/30
119/119 [==============================] - 12s 103ms/step - loss: 2.0811 - accuracy: 0.3759 - val_loss: 2.1016 - val_accuracy: 0.3689
Epoch 11/30
119/119 [==============================] - 12s 102ms/step - loss: 2.0393 - accuracy: 0.3956 - val_loss: 2.0799 - val_accuracy: 0.3679
Epoch 12/30
119/119 [==============================] - 12s 103ms/step - loss: 2.0088 - accuracy: 0.3985 - val_loss: 2.0477 - val_accuracy: 0.3668
Epoch 13/30
119/119 [==============================] - 12s 102ms/step - loss: 1.9777 - accuracy: 0.4102 - val_loss: 2.0073 - val_accuracy: 0.3932
Epoch 14/30
119/119 [==============================] - 12s 104ms/step - loss: 1.9482 - accuracy: 0.4188 - val_loss: 2.0031 - val_accuracy: 0.3901
Epoch 15/30
119/119 [==============================] - 12s 102ms/step - loss: 1.9236 - accuracy: 0.4222 - val_loss: 1.9815 - val_accuracy: 0.3975
Epoch 16/30
119/119 [==============================] - 12s 102ms/step - loss: 1.8935 - accuracy: 0.4388 - val_loss: 1.9787 - val_accuracy: 0.4006
Epoch 17/30
119/119 [==============================] - 12s 101ms/step - loss: 1.8654 - accuracy: 0.4388 - val_loss: 1.9359 - val_accuracy: 0.3901
Epoch 18/30
119/119 [==============================] - 13s 108ms/step - loss: 1.8502 - accuracy: 0.4496 - val_loss: 1.9298 - val_accuracy: 0.4027
Epoch 19/30
119/119 [==============================] - 12s 105ms/step - loss: 1.8193 - accuracy: 0.4563 - val_loss: 1.9242 - val_accuracy: 0.4133
Epoch 20/30
119/119 [==============================] - 12s 102ms/step - loss: 1.7988 - accuracy: 0.4578 - val_loss: 1.9184 - val_accuracy: 0.4123
Epoch 21/30
119/119 [==============================] - 12s 100ms/step - loss: 1.7751 - accuracy: 0.4689 - val_loss: 1.8955 - val_accuracy: 0.4207
Epoch 22/30
119/119 [==============================] - 12s 100ms/step - loss: 1.7590 - accuracy: 0.4742 - val_loss: 1.9157 - val_accuracy: 0.4112
Epoch 23/30
119/119 [==============================] - 12s 102ms/step - loss: 1.7343 - accuracy: 0.4803 - val_loss: 1.8682 - val_accuracy: 0.4260
Epoch 24/30
119/119 [==============================] - 12s 103ms/step - loss: 1.7198 - accuracy: 0.4889 - val_loss: 1.8609 - val_accuracy: 0.4323
Epoch 25/30
119/119 [==============================] - 12s 101ms/step - loss: 1.6944 - accuracy: 0.4956 - val_loss: 1.8286 - val_accuracy: 0.4313
Epoch 26/30
119/119 [==============================] - 12s 100ms/step - loss: 1.6771 - accuracy: 0.4999 - val_loss: 1.8198 - val_accuracy: 0.4419
Epoch 27/30
119/119 [==============================] - 12s 102ms/step - loss: 1.6619 - accuracy: 0.4992 - val_loss: 1.8172 - val_accuracy: 0.4429
Epoch 28/30
119/119 [==============================] - 12s 101ms/step - loss: 1.6442 - accuracy: 0.5086 - val_loss: 1.8119 - val_accuracy: 0.4387
Epoch 29/30
119/119 [==============================] - 12s 102ms/step - loss: 1.6212 - accuracy: 0.5103 - val_loss: 1.7779 - val_accuracy: 0.4493
Epoch 30/30
119/119 [==============================] - 12s 99ms/step - loss: 1.6093 - accuracy: 0.5168 - val_loss: 1.8065 - val_accuracy: 0.4387
15/15 [==============================] - 1s 33ms/step - loss: 1.7299 - accuracy: 0.4704

WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.

(None, 32, 32, 3) (None, 32, 32, 16)
(None, 32, 32, 16)
(None, 32, 32, 16) (None, 32, 32, 32)
(None, 32, 32, 32)

WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.
WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.

(None, 32, 32, 32) (None, 32, 32, 64)
(None, 32, 32, 64)
Epoch 1/30
119/119 [==============================] - 45s 377ms/step - loss: 2.5993 - accuracy: 0.2206 - val_loss: 3.0598 - val_accuracy: 0.0835
Epoch 2/30
119/119 [==============================] - 45s 374ms/step - loss: 2.3132 - accuracy: 0.2948 - val_loss: 2.9004 - val_accuracy: 0.1332
Epoch 3/30
119/119 [==============================] - 31s 256ms/step - loss: 2.1634 - accuracy: 0.3406 - val_loss: 2.5185 - val_accuracy: 0.2082
Epoch 4/30
119/119 [==============================] - 29s 244ms/step - loss: 2.0424 - accuracy: 0.3853 - val_loss: 2.3367 - val_accuracy: 0.2865
Epoch 5/30
119/119 [==============================] - 29s 247ms/step - loss: 1.9609 - accuracy: 0.4034 - val_loss: 2.2986 - val_accuracy: 0.3214
Epoch 6/30
119/119 [==============================] - 31s 260ms/step - loss: 1.8778 - accuracy: 0.4247 - val_loss: 2.6598 - val_accuracy: 0.2569
Epoch 7/30
119/119 [==============================] - 32s 265ms/step - loss: 1.8133 - accuracy: 0.4439 - val_loss: 2.0327 - val_accuracy: 0.3658
Epoch 8/30
119/119 [==============================] - 33s 276ms/step - loss: 1.7591 - accuracy: 0.4550 - val_loss: 2.1733 - val_accuracy: 0.3478
Epoch 9/30
119/119 [==============================] - 30s 252ms/step - loss: 1.6747 - accuracy: 0.4794 - val_loss: 2.1670 - val_accuracy: 0.3573
Epoch 10/30
119/119 [==============================] - 30s 253ms/step - loss: 1.6190 - accuracy: 0.4995 - val_loss: 1.9513 - val_accuracy: 0.3964
Epoch 11/30
119/119 [==============================] - 30s 254ms/step - loss: 1.5573 - accuracy: 0.5228 - val_loss: 1.8114 - val_accuracy: 0.4461
Epoch 12/30
119/119 [==============================] - 30s 255ms/step - loss: 1.4940 - accuracy: 0.5333 - val_loss: 1.8698 - val_accuracy: 0.4313
Epoch 13/30
119/119 [==============================] - 30s 255ms/step - loss: 1.4651 - accuracy: 0.5507 - val_loss: 2.1673 - val_accuracy: 0.3552
Epoch 14/30
119/119 [==============================] - 31s 257ms/step - loss: 1.4159 - accuracy: 0.5569 - val_loss: 1.9580 - val_accuracy: 0.4260
Epoch 15/30
119/119 [==============================] - 31s 261ms/step - loss: 1.3878 - accuracy: 0.5684 - val_loss: 1.7248 - val_accuracy: 0.4630
Epoch 16/30
119/119 [==============================] - 31s 257ms/step - loss: 1.3594 - accuracy: 0.5738 - val_loss: 2.0013 - val_accuracy: 0.3858
Epoch 17/30
119/119 [==============================] - 31s 260ms/step - loss: 1.3107 - accuracy: 0.5922 - val_loss: 1.7925 - val_accuracy: 0.4567
Epoch 18/30
119/119 [==============================] - 31s 260ms/step - loss: 1.2900 - accuracy: 0.5950 - val_loss: 1.7052 - val_accuracy: 0.4778
Epoch 19/30
119/119 [==============================] - 31s 257ms/step - loss: 1.2597 - accuracy: 0.6093 - val_loss: 1.7073 - val_accuracy: 0.4609
Epoch 20/30
119/119 [==============================] - 31s 258ms/step - loss: 1.2434 - accuracy: 0.6165 - val_loss: 1.8809 - val_accuracy: 0.4588
Epoch 21/30
119/119 [==============================] - 31s 257ms/step - loss: 1.1844 - accuracy: 0.6290 - val_loss: 1.9237 - val_accuracy: 0.4662
Epoch 22/30
119/119 [==============================] - 31s 259ms/step - loss: 1.1665 - accuracy: 0.6351 - val_loss: 1.5667 - val_accuracy: 0.5021
Epoch 23/30
119/119 [==============================] - 31s 258ms/step - loss: 1.1333 - accuracy: 0.6464 - val_loss: 1.7319 - val_accuracy: 0.4884
Epoch 24/30
119/119 [==============================] - 31s 259ms/step - loss: 1.1148 - accuracy: 0.6521 - val_loss: 1.7901 - val_accuracy: 0.4725
Epoch 25/30
119/119 [==============================] - 32s 271ms/step - loss: 1.0952 - accuracy: 0.6693 - val_loss: 1.9127 - val_accuracy: 0.4355
Epoch 26/30
119/119 [==============================] - 33s 276ms/step - loss: 1.0709 - accuracy: 0.6661 - val_loss: 1.9725 - val_accuracy: 0.4313
Epoch 27/30
119/119 [==============================] - 31s 261ms/step - loss: 1.0568 - accuracy: 0.6758 - val_loss: 1.6910 - val_accuracy: 0.5137
Epoch 28/30
119/119 [==============================] - 31s 259ms/step - loss: 1.0248 - accuracy: 0.6841 - val_loss: 1.8256 - val_accuracy: 0.4789
Epoch 29/30
119/119 [==============================] - 31s 259ms/step - loss: 1.0102 - accuracy: 0.6840 - val_loss: 1.9483 - val_accuracy: 0.4535
Epoch 30/30
119/119 [==============================] - 31s 260ms/step - loss: 0.9965 - accuracy: 0.6919 - val_loss: 1.6930 - val_accuracy: 0.5095
15/15 [==============================] - 1s 76ms/step - loss: 1.5292 - accuracy: 0.5645


print (best_config)

(4, True, True, 0.001, 0.5877378582954407)


from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import EarlyStopping


class YourModel(DefaultModel):
    def __init__(self,num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose,
                 name='network1',
                 width=32, height=32, depth=3,
                 num_classes=20, 
                 is_augmentation = False,
                 activation_func='relu',
                 optimizer='adam',
                 batch_size=32,
                 num_epochs= 20):
        super(YourModel, self).__init__(name, width, height, depth, num_classes, is_augmentation, 
                                        activation_func, optimizer, batch_size, num_epochs, 
                                        learning_rate, verbose)
        self.num_channels = num_channels
        self.mean_pool = mean_pool
        self.batch_norm = batch_norm
        self.use_skip = use_skip
        self.blocks = blocks
    
    
    def build_cnn(self,x):
        #Insert your code here
        x1 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x)
        if self.batch_norm:
            x1 = layers.BatchNormalization() (x1)
        x1 = layers.Activation('relu') (x1)
        x2 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x1)
        if self.batch_norm:
                x2 = layers.BatchNormalization()(x2)
        if x.shape != x2.shape:
            if x2.shape[3] > x.shape[3]:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x2.shape[3] - x.shape[3] , 0]])
                x = tf.pad(x, pad_tns, mode ='CONSTANT', constant_values=0)
            else:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x.shape[3] - x2.shape[3] , 0]])
                x = tf.pad(x2, pad_tns, mode ='CONSTANT', constant_values=0)
        x2_skip = layers.add([x, x2])
        x2_skip = layers.Activation('relu')(x2_skip)
        if self.mean_pool:
            output_layer = layers.AveragePooling2D(pool_size=(2, 2), padding='same')(x2_skip)
        else: 
            output_layer = x2_skip
        return output_layer
    
    def build_resnet(self):
        self.input_layer = layers.Input(shape=(self.width, self.height, self.depth))
        x = self.input_layer
        for i in range (self.blocks):
            x = self.build_cnn(x)
            self.num_channels = self.num_channels*2
        output_layer = GlobalAveragePooling2D()(x)
        output_layer = layers.Flatten()(output_layer)
        output_layer = layers.Dense(self.num_classes, activation='softmax')(output_layer)
        self.model = tf.keras.models.Model(inputs=self.input_layer, outputs=output_layer)
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    def fit(self, data_manager, batch_size=None, num_epochs=None):
        #Insert your code here
        batch_size = self.batch_size if batch_size is None else batch_size
        num_epochs = self.num_epochs if num_epochs is None else num_epochs
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
        early_stopping = EarlyStopping(monitor='val_accuracy', patience=3)
        callbacks = [early_stopping]
        if self.is_augmentation == True:
            datagen = tf.keras.preprocessing.image.ImageDataGenerator(
                width_shift_range=0.05,
                zoom_range = 0.05,
                rotation_range=5
            )
            datagen.fit(data_manager.X_train)
            it = datagen.flow(data_manager.X_train,data_manager.y_train, shuffle = True, batch_size =batch_size)
            self.history = self.model.fit(x = it, 
                          validation_data = (data_manager.X_valid, data_manager.y_valid), 
                          epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)
        else:
            self.history = self.model.fit(x = data_manager.X_train, y = data_manager.y_train,
                                      validation_data = (data_manager.X_valid, data_manager.y_valid), 
                                      epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)


#Insert your code here. You can add more cells if necessary
testModel = YourModel(32,4, True, True,True, 0.001,True)
testModel.build_resnet()
testModel.fit(data_manager, batch_size = 16, num_epochs = 30)
testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)

Epoch 1/30
473/473 [==============================] - 20s 41ms/step - loss: 2.5141 - accuracy: 0.2286 - val_loss: 2.5227 - val_accuracy: 0.2442
Epoch 2/30
473/473 [==============================] - 20s 41ms/step - loss: 2.1007 - accuracy: 0.3468 - val_loss: 2.0654 - val_accuracy: 0.3668
Epoch 3/30
473/473 [==============================] - 20s 42ms/step - loss: 1.8436 - accuracy: 0.4198 - val_loss: 1.8843 - val_accuracy: 0.4228
Epoch 4/30
473/473 [==============================] - 20s 41ms/step - loss: 1.6569 - accuracy: 0.4819 - val_loss: 1.7000 - val_accuracy: 0.4662
Epoch 5/30
473/473 [==============================] - 19s 41ms/step - loss: 1.4680 - accuracy: 0.5409 - val_loss: 1.7330 - val_accuracy: 0.4746
Epoch 6/30
473/473 [==============================] - 20s 43ms/step - loss: 1.3347 - accuracy: 0.5811 - val_loss: 1.6279 - val_accuracy: 0.5148
Epoch 7/30
473/473 [==============================] - 19s 41ms/step - loss: 1.2063 - accuracy: 0.6224 - val_loss: 1.5319 - val_accuracy: 0.5423
Epoch 8/30
473/473 [==============================] - 20s 42ms/step - loss: 1.0968 - accuracy: 0.6548 - val_loss: 1.5071 - val_accuracy: 0.5497
Epoch 9/30
473/473 [==============================] - 19s 41ms/step - loss: 0.9708 - accuracy: 0.6910 - val_loss: 1.7288 - val_accuracy: 0.5349
Epoch 10/30
473/473 [==============================] - 19s 40ms/step - loss: 0.8555 - accuracy: 0.7279 - val_loss: 1.7070 - val_accuracy: 0.5201
Epoch 11/30
473/473 [==============================] - 19s 40ms/step - loss: 0.7318 - accuracy: 0.7672 - val_loss: 1.5414 - val_accuracy: 0.5698
Epoch 12/30
473/473 [==============================] - 19s 40ms/step - loss: 0.6186 - accuracy: 0.8026 - val_loss: 1.4291 - val_accuracy: 0.6099
Epoch 13/30
473/473 [==============================] - 19s 41ms/step - loss: 0.5340 - accuracy: 0.8303 - val_loss: 1.6177 - val_accuracy: 0.5793
Epoch 14/30
473/473 [==============================] - 19s 40ms/step - loss: 0.4371 - accuracy: 0.8618 - val_loss: 1.6358 - val_accuracy: 0.6004
Epoch 15/30
473/473 [==============================] - 19s 40ms/step - loss: 0.3581 - accuracy: 0.8847 - val_loss: 1.7178 - val_accuracy: 0.5698
15/15 [==============================] - 0s 20ms/step - loss: 1.6682 - accuracy: 0.5751

0.5750528573989868


testModel.plot_progress()
testModel.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

30/30 [==============================] - 0s 11ms/step

<Figure size 640x480 with 0 Axes>


testModel.model.save('models/augmentation_true_model.h5')


class YourModel(DefaultModel):
    def __init__(self,num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose,
                 name='network1',
                 width=32, height=32, depth=3,
                 num_classes=20, 
                 is_augmentation = False,
                 activation_func='relu',
                 optimizer='adam',
                 batch_size=32,
                 num_epochs= 20):
        super(YourModel, self).__init__(name, width, height, depth, num_classes, is_augmentation, 
                                        activation_func, optimizer, batch_size, num_epochs, 
                                        learning_rate, verbose)
        self.num_channels = num_channels
        self.mean_pool = mean_pool
        self.batch_norm = batch_norm
        self.use_skip = use_skip
        self.blocks = blocks
    
    
    def build_cnn(self,x):
        #Insert your code here
        x1 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x)
        if self.batch_norm:
            x1 = layers.BatchNormalization() (x1)
        x1 = layers.Activation('relu') (x1)
        x2 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x1)
        if self.batch_norm:
                x2 = layers.BatchNormalization()(x2)
        if x.shape != x2.shape:
            if x2.shape[3] > x.shape[3]:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x2.shape[3] - x.shape[3] , 0]])
                x = tf.pad(x, pad_tns, mode ='CONSTANT', constant_values=0)
            else:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x.shape[3] - x2.shape[3] , 0]])
                x = tf.pad(x2, pad_tns, mode ='CONSTANT', constant_values=0)
        x2_skip = layers.add([x, x2])
        x2_skip = layers.Activation('relu')(x2_skip)
        if self.mean_pool:
            output_layer = layers.AveragePooling2D(pool_size=(2, 2), padding='same')(x2_skip)
        else: 
            output_layer = x2_skip
        return output_layer
    
    def build_resnet(self):
        self.input_layer = layers.Input(shape=(self.width, self.height, self.depth))
        x = self.input_layer
        for i in range (self.blocks):
            x = self.build_cnn(x)
            self.num_channels = self.num_channels*2
        output_layer = GlobalAveragePooling2D()(x)
        output_layer = layers.Flatten()(output_layer)
        output_layer = layers.Dense(self.num_classes, activation='softmax')(output_layer)
        self.model = tf.keras.models.Model(inputs=self.input_layer, outputs=output_layer)
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    def fit(self, data_manager, batch_size=None, num_epochs=None):
        #Insert your code here
        batch_size = self.batch_size if batch_size is None else batch_size
        num_epochs = self.num_epochs if num_epochs is None else num_epochs
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
        early_stopping = EarlyStopping(monitor='val_accuracy', patience=3)
        callbacks = [early_stopping]
        if self.is_augmentation == True:
            datagen = tf.keras.preprocessing.image.ImageDataGenerator(
                brightness_range=(0.9, 1.1),
                horizontal_flip=True,
                width_shift_range=0.2,
                height_shift_range=0.1,
                rotation_range= 10,
                zoom_range = 0.1
            )
            datagen.fit(data_manager.X_train)
            it = datagen.flow(data_manager.X_train,data_manager.y_train, shuffle = False, batch_size =batch_size)
            self.history = self.model.fit(x = it, 
                          validation_data = (data_manager.X_valid, data_manager.y_valid), 
                          epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)
        else:
            self.history = self.model.fit(x = data_manager.X_train, y = data_manager.y_train,
                                      validation_data = (data_manager.X_valid, data_manager.y_valid), 
                                      epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)


testModel = YourModel(32,4, True, True,True, 0.001,True)
testModel.build_resnet()
testModel.fit(data_manager, batch_size = 16, num_epochs = 20)
testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)

Epoch 1/20
473/473 [==============================] - 21s 44ms/step - loss: 2.4680 - accuracy: 0.2439 - val_loss: 2.4805 - val_accuracy: 0.2463
Epoch 2/20
473/473 [==============================] - 20s 43ms/step - loss: 2.0364 - accuracy: 0.3698 - val_loss: 2.0306 - val_accuracy: 0.3837
Epoch 3/20
473/473 [==============================] - 20s 42ms/step - loss: 1.7709 - accuracy: 0.4516 - val_loss: 1.8833 - val_accuracy: 0.4397
Epoch 4/20
473/473 [==============================] - 20s 42ms/step - loss: 1.5909 - accuracy: 0.5071 - val_loss: 1.7732 - val_accuracy: 0.4545
Epoch 5/20
473/473 [==============================] - 20s 41ms/step - loss: 1.4055 - accuracy: 0.5636 - val_loss: 1.5745 - val_accuracy: 0.5095
Epoch 6/20
473/473 [==============================] - 19s 41ms/step - loss: 1.2683 - accuracy: 0.6052 - val_loss: 1.5717 - val_accuracy: 0.5148
Epoch 7/20
473/473 [==============================] - 20s 42ms/step - loss: 1.1440 - accuracy: 0.6467 - val_loss: 1.5202 - val_accuracy: 0.5613
Epoch 8/20
473/473 [==============================] - 20s 42ms/step - loss: 1.0157 - accuracy: 0.6758 - val_loss: 1.4315 - val_accuracy: 0.5645
Epoch 9/20
473/473 [==============================] - 20s 42ms/step - loss: 0.8867 - accuracy: 0.7190 - val_loss: 1.9850 - val_accuracy: 0.4799
Epoch 10/20
473/473 [==============================] - 20s 42ms/step - loss: 0.7710 - accuracy: 0.7516 - val_loss: 1.5861 - val_accuracy: 0.5455
Epoch 11/20
473/473 [==============================] - 20s 42ms/step - loss: 0.6408 - accuracy: 0.7909 - val_loss: 1.6589 - val_accuracy: 0.5793
Epoch 12/20
473/473 [==============================] - 20s 42ms/step - loss: 0.5228 - accuracy: 0.8315 - val_loss: 1.7419 - val_accuracy: 0.5603
Epoch 13/20
473/473 [==============================] - 20s 42ms/step - loss: 0.4382 - accuracy: 0.8575 - val_loss: 1.7577 - val_accuracy: 0.5687
Epoch 14/20
473/473 [==============================] - 20s 41ms/step - loss: 0.3249 - accuracy: 0.8971 - val_loss: 2.1832 - val_accuracy: 0.5233
15/15 [==============================] - 0s 19ms/step - loss: 2.2280 - accuracy: 0.5211

0.5211416482925415


testModel.plot_progress()
testModel.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

30/30 [==============================] - 0s 16ms/step

<Figure size 640x480 with 0 Axes>


testModel.model.save('models/augmentation_false_model.h5')


sp = SimplePreprocessor(width=32, height=32)
data_manager = DatasetManager([sp])
data_manager.load(label_folder_dict, verbose=100)
data_manager.process_data_label()
data_manager.train_valid_test_split()

birds 512
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
bottles 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
breads 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
butterfiles 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cakes 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
cats 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
chickens 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cows 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
dogs 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
ducks 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
elephants 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
fishes 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
handguns 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
horses 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lions 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lipsticks 400
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
seals 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
snakes 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
spiders 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
vases 368
Processed 100/500
Processed 200/500
Processed 300/500


from keras.utils import to_categorical
import random

def mixup(data_manager,batch_size,alpha=0.2):
    l = len(data_manager.X_train)
    mixed_data = []
    mixed_labels = []
    lam = np.random.beta(alpha,alpha)
    print ("shape before mixup: X: ",data_manager.X_train.shape," y: ",data_manager.y_train.shape)
    for i in range(0, l, 4):
        if i + 1 < l:
            x1, y1 = data_manager.X_train[i], data_manager.y_train[i]
            x2, y2 = data_manager.X_train[i + 1], data_manager.y_train[i + 1]
            # Generate random mixing coefficient from a beta distribution
            lam = np.random.beta(alpha,alpha)
            mixed_x = (lam * x1) + ((1 - lam) * x2)
            mixed_y = (lam * y1) + ((1 - lam) * y2)
            mixed_data.append(mixed_x)
            mixed_labels.append(mixed_y)

    data_manager.X_train = np.concatenate((data_manager.X_train, np.array(mixed_data)))
    data_manager.y_train = np.concatenate((data_manager.y_train,np.array(mixed_labels)))


class YourModel(DefaultModel):
    def __init__(self,num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose,
                 name='network1',
                 width=32, height=32, depth=3,
                 num_classes=20, 
                 is_augmentation = True,
                 activation_func='relu',
                 optimizer='adam',
                 batch_size=32,
                 num_epochs= 20):
        super(YourModel, self).__init__(name, width, height, depth, num_classes, is_augmentation, 
                                        activation_func, optimizer, batch_size, num_epochs, 
                                        learning_rate, verbose)
        self.num_channels = num_channels
        self.mean_pool = mean_pool
        self.batch_norm = batch_norm
        self.use_skip = use_skip
        self.blocks = blocks
    
    
    def build_cnn(self,x):
        #Insert your code here
        x1 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x)
        if self.batch_norm:
            x1 = layers.BatchNormalization() (x1)
        x1 = layers.Activation('relu') (x1)
        x2 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x1)
        if self.batch_norm:
                x2 = layers.BatchNormalization()(x2)
        if x.shape != x2.shape:
            if x2.shape[3] > x.shape[3]:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x2.shape[3] - x.shape[3] , 0]])
                x = tf.pad(x, pad_tns, mode ='CONSTANT', constant_values=0)
            else:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x.shape[3] - x2.shape[3] , 0]])
                x = tf.pad(x2, pad_tns, mode ='CONSTANT', constant_values=0)
        x2_skip = layers.add([x, x2])
        x2_skip = layers.Activation('relu')(x2_skip)
        if self.mean_pool:
            output_layer = layers.AveragePooling2D(pool_size=(2, 2), padding='same')(x2_skip)
        else: 
            output_layer = x2_skip
        return output_layer
    
    def build_resnet(self):
        self.input_layer = layers.Input(shape=(self.width, self.height, self.depth))
        x = self.input_layer
        for i in range (self.blocks):
            x = self.build_cnn(x)
            self.num_channels = self.num_channels*2
        output_layer = GlobalAveragePooling2D()(x)
        output_layer = layers.Flatten()(output_layer)
        output_layer = layers.Dense(self.num_classes, activation='softmax')(output_layer)
        self.model = tf.keras.models.Model(inputs=self.input_layer, outputs=output_layer)
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
   
    def fit(self, data_manager, batch_size=None, num_epochs=None):
        #Insert your code here
        batch_size = self.batch_size if batch_size is None else batch_size
        num_epochs = self.num_epochs if num_epochs is None else num_epochs
        self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy'])
        early_stopping = EarlyStopping(monitor='val_accuracy', patience=3)
        callbacks = [early_stopping]
        if self.is_augmentation == True:
            datagen = tf.keras.preprocessing.image.ImageDataGenerator(
                width_shift_range=0.05,
                zoom_range = 0.05,
            )
            datagen.fit(data_manager.X_train)
            it = datagen.flow(data_manager.X_train,data_manager.y_train, shuffle = False, batch_size =batch_size)
            self.history = self.model.fit(x = it, 
                          validation_data = (data_manager.X_valid, data_manager.y_valid), 
                          epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)
        else:
            self.history = self.model.fit(x = data_manager.X_train, y = data_manager.y_train,
                                      validation_data = (data_manager.X_valid, data_manager.y_valid), 
                                      epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)


testModel = YourModel(16,5, True, True,True, 0.001,True)
testModel.build_resnet()
mixup(data_manager,16)
testModel.fit(data_manager, batch_size = 16, num_epochs = 20)
testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)

shape before mixup: X:  (7560, 32, 32, 3)  y:  (7560,)
Epoch 1/20

C:\Users\manut\anaconda3\envs\gpu\lib\site-packages\tensorflow\python\data\ops\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.
  warnings.warn(

591/591 [==============================] - 30s 51ms/step - loss: 2.6239 - accuracy: 0.1894 - val_loss: 2.4572 - val_accuracy: 0.2717
Epoch 2/20
591/591 [==============================] - 30s 50ms/step - loss: 2.3045 - accuracy: 0.2848 - val_loss: 2.2291 - val_accuracy: 0.3044
Epoch 3/20
591/591 [==============================] - 31s 52ms/step - loss: 2.1120 - accuracy: 0.3287 - val_loss: 1.8786 - val_accuracy: 0.4355
Epoch 4/20
591/591 [==============================] - 34s 57ms/step - loss: 1.9323 - accuracy: 0.3872 - val_loss: 2.2891 - val_accuracy: 0.3562
Epoch 5/20
591/591 [==============================] - 34s 57ms/step - loss: 1.7978 - accuracy: 0.4207 - val_loss: 1.7680 - val_accuracy: 0.4778
Epoch 6/20
591/591 [==============================] - 33s 56ms/step - loss: 1.6581 - accuracy: 0.4618 - val_loss: 1.7923 - val_accuracy: 0.4715
Epoch 7/20
591/591 [==============================] - 34s 58ms/step - loss: 1.5390 - accuracy: 0.4896 - val_loss: 1.9705 - val_accuracy: 0.4556
Epoch 8/20
591/591 [==============================] - 34s 58ms/step - loss: 1.3993 - accuracy: 0.5232 - val_loss: 1.7156 - val_accuracy: 0.4915
Epoch 9/20
591/591 [==============================] - 35s 59ms/step - loss: 1.2768 - accuracy: 0.5550 - val_loss: 1.7433 - val_accuracy: 0.5021
Epoch 10/20
591/591 [==============================] - 34s 58ms/step - loss: 1.1250 - accuracy: 0.5878 - val_loss: 1.7705 - val_accuracy: 0.5148
Epoch 11/20
591/591 [==============================] - 34s 57ms/step - loss: 0.9957 - accuracy: 0.6177 - val_loss: 1.8656 - val_accuracy: 0.5011
Epoch 12/20
591/591 [==============================] - 34s 58ms/step - loss: 0.8425 - accuracy: 0.6492 - val_loss: 1.8326 - val_accuracy: 0.5285
Epoch 13/20
591/591 [==============================] - 34s 58ms/step - loss: 0.7374 - accuracy: 0.6666 - val_loss: 2.0426 - val_accuracy: 0.4937
Epoch 14/20
591/591 [==============================] - 35s 59ms/step - loss: 0.6287 - accuracy: 0.6916 - val_loss: 2.2982 - val_accuracy: 0.4820
Epoch 15/20
591/591 [==============================] - 35s 59ms/step - loss: 0.5435 - accuracy: 0.7114 - val_loss: 2.0785 - val_accuracy: 0.5285
15/15 [==============================] - 0s 24ms/step - loss: 2.3560 - accuracy: 0.4831

0.4830866754055023


testModel.plot_progress()
testModel.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

30/30 [==============================] - 0s 15ms/step

<Figure size 640x480 with 0 Axes>


testModel.model.save('models/data_mixup_model.h5')


tf.config.run_functions_eagerly(True)    
#Insert your code here. You can add more cells if necessary
class OVA_loss (tf.keras.losses.Loss):
    def __init__(self,eps=1E-10, num_classes = 20):
        super(OVA_loss, self).__init__()
        self.eps = eps
        self.num_classes = num_classes

    def call(self, y_true, y_pred):

        y_true_1_hot = tf.one_hot(tf.transpose(tf.cast(y_true, tf.int32), perm= [1,0])[0], depth= self.num_classes, axis=-1)
        loss_true_class = -tf.math.log(y_pred + self.eps) * y_true_1_hot
        # Find the index of the true class
        true_class_index = tf.argmax(y_true_1_hot, axis=-1)
        # Create a mask for the true class
        mask = tf.one_hot(true_class_index, depth= self.num_classes, on_value=1.0, off_value=0.0)
        # Invert the mask (0s become 1s, and 1s become 0s)
        y_true_modified = 1 - mask
                
        # Compute the negative log-likelihood for the other class
        loss_other_class = -tf.math.log(1.0 - y_pred + self.eps) * y_true_modified

        # Combine the losses for both classes
        total_loss = loss_true_class + loss_other_class

        return total_loss

class YourModel(DefaultModel):
    def __init__(self,num_channels,blocks,mean_pool,batch_norm,use_skip,learning_rate,verbose,
                 name='network1',
                 width=32, height=32, depth=3,
                 num_classes=20, 
                 is_augmentation = True,
                 activation_func='relu',
                 optimizer='adam',
                 batch_size=32,
                 num_epochs= 20):
        super(YourModel, self).__init__(name, width, height, depth, num_classes, is_augmentation, 
                                        activation_func, optimizer, batch_size, num_epochs, 
                                        learning_rate, verbose)
        self.num_channels = num_channels
        self.mean_pool = mean_pool
        self.batch_norm = batch_norm
        self.use_skip = use_skip
        self.blocks = blocks
    
    
    def build_cnn(self,x):
        #Insert your code here
        x1 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x)
        if self.batch_norm:
            x1 = layers.BatchNormalization() (x1)
        x1 = layers.Activation('relu') (x1)
        x2 = layers.Conv2D(self.num_channels, (3, 3), strides=(1, 1), padding='same')(x1)
        if self.batch_norm:
                x2 = layers.BatchNormalization()(x2)
        if x.shape != x2.shape:
            if x2.shape[3] > x.shape[3]:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x2.shape[3] - x.shape[3] , 0]])
                x = tf.pad(x, pad_tns, mode ='CONSTANT', constant_values=0)
            else:
                pad_tns = tf.constant([[0,0],[0, 0] ,[0,0], [x.shape[3] - x2.shape[3] , 0]])
                x = tf.pad(x2, pad_tns, mode ='CONSTANT', constant_values=0)
        x2_skip = layers.add([x, x2])
        x2_skip = layers.Activation('relu')(x2_skip)
        if self.mean_pool:
            output_layer = layers.AveragePooling2D(pool_size=(2, 2), padding='same')(x2_skip)
        else: 
            output_layer = x2_skip
        return output_layer
    
    def build_resnet(self):
        self.input_layer = layers.Input(shape=(self.width, self.height, self.depth))
        x = self.input_layer
        for i in range (self.blocks):
            x = self.build_cnn(x)
            self.num_channels = self.num_channels*2
        output_layer = GlobalAveragePooling2D()(x)
        output_layer = layers.Flatten()(output_layer)
        output_layer = layers.Dense(self.num_classes, activation='sigmoid')(output_layer)
        self.model = tf.keras.models.Model(inputs=self.input_layer, outputs=output_layer)
        self.model.compile(optimizer=self.optimizer, loss= OVA_loss(), metrics=['accuracy'])
   
    def fit(self, data_manager, batch_size=None, num_epochs=None):
        #Insert your code here
        batch_size = self.batch_size if batch_size is None else batch_size
        num_epochs = self.num_epochs if num_epochs is None else num_epochs
        self.model.compile(optimizer=self.optimizer, loss= OVA_loss(), metrics=['accuracy'])
        early_stopping = EarlyStopping(monitor='val_accuracy', patience=6)
        callbacks = [early_stopping]
        if self.is_augmentation == True:
            datagen = tf.keras.preprocessing.image.ImageDataGenerator(
                brightness_range=(0.9, 1.1),
                width_shift_range=0.2,
                height_shift_range=0.1,
                rotation_range= 10,
            )
            datagen.fit(data_manager.X_train)
            it = datagen.flow(data_manager.X_train,data_manager.y_train, shuffle = False, batch_size =batch_size)
            self.history = self.model.fit(x = it, 
                          validation_data = (data_manager.X_valid, data_manager.y_valid), 
                          epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)
        else:
            self.history = self.model.fit(x = data_manager.X_train, y = data_manager.y_train,
                                      validation_data = (data_manager.X_valid, data_manager.y_valid), 
                                      epochs = num_epochs, batch_size = batch_size, callbacks = callbacks,verbose= self.verbose)


testModel = YourModel(32,4, True, True,True, 0.001,True)
testModel.build_resnet()
mixup(data_manager,16)
testModel.fit(data_manager, batch_size = 32, num_epochs = 40)
testModel.compute_accuracy(data_manager.X_test, data_manager.y_test)

shape before mixup: X:  (7560, 32, 32, 3)  y:  (7560,)
Epoch 1/40

C:\Users\manut\anaconda3\envs\gpu\lib\site-packages\tensorflow\python\data\ops\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.
  warnings.warn(

355/355 [==============================] - 22s 54ms/step - loss: 0.1922 - accuracy: 0.1469 - val_loss: 0.1752 - val_accuracy: 0.2548
Epoch 2/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1721 - accuracy: 0.2139 - val_loss: 0.1638 - val_accuracy: 0.2854
Epoch 3/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1658 - accuracy: 0.2498 - val_loss: 0.1596 - val_accuracy: 0.3034
Epoch 4/40
355/355 [==============================] - 19s 53ms/step - loss: 0.1597 - accuracy: 0.2867 - val_loss: 0.1429 - val_accuracy: 0.3953
Epoch 5/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1539 - accuracy: 0.3161 - val_loss: 0.1389 - val_accuracy: 0.4228
Epoch 6/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1490 - accuracy: 0.3385 - val_loss: 0.1300 - val_accuracy: 0.4693
Epoch 7/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1441 - accuracy: 0.3597 - val_loss: 0.1359 - val_accuracy: 0.4313
Epoch 8/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1389 - accuracy: 0.3847 - val_loss: 0.1400 - val_accuracy: 0.4397
Epoch 9/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1349 - accuracy: 0.4059 - val_loss: 0.1252 - val_accuracy: 0.5032
Epoch 10/40
355/355 [==============================] - 19s 55ms/step - loss: 0.1305 - accuracy: 0.4248 - val_loss: 0.1472 - val_accuracy: 0.4207
Epoch 11/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1261 - accuracy: 0.4369 - val_loss: 0.1267 - val_accuracy: 0.5021
Epoch 12/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1214 - accuracy: 0.4577 - val_loss: 0.1312 - val_accuracy: 0.4937
Epoch 13/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1175 - accuracy: 0.4652 - val_loss: 0.1252 - val_accuracy: 0.5349
Epoch 14/40
355/355 [==============================] - 19s 55ms/step - loss: 0.1124 - accuracy: 0.4874 - val_loss: 0.1533 - val_accuracy: 0.4524
Epoch 15/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1086 - accuracy: 0.4976 - val_loss: 0.1320 - val_accuracy: 0.5275
Epoch 16/40
355/355 [==============================] - 19s 54ms/step - loss: 0.1035 - accuracy: 0.5130 - val_loss: 0.1427 - val_accuracy: 0.5116
Epoch 17/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0995 - accuracy: 0.5309 - val_loss: 0.1481 - val_accuracy: 0.4958
Epoch 18/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0948 - accuracy: 0.5362 - val_loss: 0.1305 - val_accuracy: 0.5412
Epoch 19/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0906 - accuracy: 0.5473 - val_loss: 0.1388 - val_accuracy: 0.5211
Epoch 20/40
355/355 [==============================] - 19s 55ms/step - loss: 0.0870 - accuracy: 0.5583 - val_loss: 0.1559 - val_accuracy: 0.4704
Epoch 21/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0825 - accuracy: 0.5648 - val_loss: 0.1473 - val_accuracy: 0.4894
Epoch 22/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0771 - accuracy: 0.5762 - val_loss: 0.1559 - val_accuracy: 0.4672
Epoch 23/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0734 - accuracy: 0.5833 - val_loss: 0.1557 - val_accuracy: 0.5159
Epoch 24/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0700 - accuracy: 0.5862 - val_loss: 0.1415 - val_accuracy: 0.5518
Epoch 25/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0657 - accuracy: 0.5994 - val_loss: 0.1450 - val_accuracy: 0.5476
Epoch 26/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0618 - accuracy: 0.6021 - val_loss: 0.1670 - val_accuracy: 0.4820
Epoch 27/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0593 - accuracy: 0.6057 - val_loss: 0.1708 - val_accuracy: 0.4503
Epoch 28/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0563 - accuracy: 0.6096 - val_loss: 0.1721 - val_accuracy: 0.4757
Epoch 29/40
355/355 [==============================] - 19s 55ms/step - loss: 0.0542 - accuracy: 0.6115 - val_loss: 0.1744 - val_accuracy: 0.4831
Epoch 30/40
355/355 [==============================] - 19s 54ms/step - loss: 0.0504 - accuracy: 0.6131 - val_loss: 0.1577 - val_accuracy: 0.5328
15/15 [==============================] - 0s 21ms/step - loss: 0.1532 - accuracy: 0.5338

0.5338266491889954


testModel.plot_progress()
testModel.plot_prediction(data_manager.X_test, data_manager.y_test, data_manager.classes)

30/30 [==============================] - 0s 11ms/step

<Figure size 640x480 with 0 Axes>


testModel.model.save('models/ova_model.h5')


sp = SimplePreprocessor(width=32, height=32)
data_manager = DatasetManager([sp])
data_manager.load(label_folder_dict, verbose=100)
data_manager.process_data_label()
data_manager.train_valid_test_split()

birds 512
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
bottles 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
breads 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
butterfiles 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cakes 432
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
cats 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
chickens 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
cows 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
dogs 501
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
ducks 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
elephants 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
fishes 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
handguns 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
horses 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lions 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
lipsticks 400
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
seals 448
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
snakes 496
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
spiders 500
Processed 100/500
Processed 200/500
Processed 300/500
Processed 400/500
Processed 500/500
vases 368
Processed 100/500
Processed 200/500
Processed 300/500


#FGSM Attack code adapted from Tut_06b
from tensorflow.keras.models import load_model
def fgsm_attack(model, input_image, input_label=None,
               epsilon=0.0313,
               clip_value_min=0.,
               clip_value_max=1.0,
               soft_label=False,
               from_logits=True):
    """
    Args:
        model: pretrained model
        input_image: original (clean) input image (tensor)
        input_label: original label (tensor, categorical representation)
        epsilon: perturbation boundary
        clip_value_min, clip_value_max: range of valid input
        from_logits = True: attack from logits otherwise attack from prediction probabilites
    Note:
        we expect the output of model should be logits vector
    """

    loss_fn = tf.keras.losses.sparse_categorical_crossentropy # compute CE loss from logits or prediction probabilities

    if type(input_image) is np.ndarray:
        input_image = tf.convert_to_tensor(input_image)

    if type(input_label) is np.ndarray:
        input_label = tf.convert_to_tensor(input_label)

    with tf.GradientTape() as tape:
        tape.watch(input_image)
        output = model(input_image)
        if not soft_label:
            loss = loss_fn(input_label, output, from_logits=from_logits) # use ground-truth label to attack
        else:
            pred_label = tf.math.argmax(output, axis=1) # use predicted label to attack
            loss = loss_fn(pred_label, output, from_logits=from_logits)

    gradient = tape.gradient(loss, input_image) # get the gradients of the loss w.r.t. the input image
    adv_image = input_image + epsilon * tf.sign(gradient) # get the final adversarial examples
    adv_image = tf.clip_by_value(adv_image, clip_value_min, clip_value_max) # clip to a valid range
    adv_image = tf.stop_gradient(adv_image) # stop the gradient to make the adversarial image as a constant input
    return adv_image


#PGD Attack code adapted from Tut_06b
def pgd_attack(model, input_image, input_label= None,
              epsilon=0.0313,
              num_steps=10,
              step_size=0.002,
              clip_value_min=0.,
              clip_value_max=1.0,
              soft_label=False,
              from_logits= False):
    """
    Args:
        model: pretrained model
        input_image: original (clean) input image (tensor)
        input_label: original label (tensor, categorical representation)
        epsilon: perturbation boundary
        num_steps: number of attack steps
        step_size: size of each move in each attack step
        clip_value_min, clip_value_max: range of valid input
        from_logits = True: attack from logits otherwise attack from prediction probabilites
    Note:
        we expect the output of model should be logits vector
    """

    loss_fn = tf.keras.losses.sparse_categorical_crossentropy  #compute CE loss from logits or prediction probabilities

    if type(input_image) is np.ndarray:
        input_image = tf.convert_to_tensor(input_image)

    if type(input_label) is np.ndarray:
        input_label = tf.convert_to_tensor(input_label)

    # random initialization around input_image
    random_noise = tf.random.uniform(shape=input_image.shape, minval=-epsilon, maxval=epsilon)
    adv_image = input_image + random_noise

    for _ in range(num_steps):
        with tf.GradientTape(watch_accessed_variables=False) as tape:
            tape.watch(adv_image)
            adv_output = model(adv_image)
            if not soft_label:
                loss = loss_fn(input_label, adv_output, from_logits= from_logits) # use ground-truth label to attack
            else:
                pred_label = tf.math.argmax(adv_output, axis=1)
                loss = loss_fn(pred_label, adv_output, from_logits= from_logits) # use predicted label to attack

        gradient = tape.gradient(loss, adv_image) # get the gradient of the loss w.r.t. the current point
        adv_image = adv_image + step_size * tf.sign(gradient) # move current adverarial example along the gradient direction with step size is eta
        adv_image = tf.clip_by_value(adv_image, input_image-epsilon, input_image+epsilon) # clip to a valid boundary
        adv_image = tf.clip_by_value(adv_image, clip_value_min, clip_value_max)  # clip to a valid range
        adv_image = tf.stop_gradient(adv_image) # stop the gradient to make the adversarial image as a constant input
    return adv_image


#MIM Attack code adapted from Tut_06b
def mim_attack(model, input_image, input_label= None,
              epsilon=0.0313,
              gamma= 0.9,
              num_steps=20,
              step_size=0.002,
              clip_value_min=0.,
              clip_value_max=1.0,
              soft_label=False,
              from_logits= True):
    """
    Args:
        model: pretrained model
        input_image: original (clean) input image (tensor)
        input_label: original label (tensor, categorical representation)
        epsilon: perturbation boundary
        gamma: momentum decay
        num_steps: number of attack steps
        step_size: size of each move in each attack step
        clip_value_min, clip_value_max: range of valid input
        from_logits = True: attack from logits otherwise attack from prediction probabilites
    Note:
        we expect the output of model should be logits vector
    """

    loss_fn = tf.keras.losses.sparse_categorical_crossentropy # compute CE loss from logits or prediction probabilities

    if type(input_image) is np.ndarray:
        input_image = tf.convert_to_tensor(input_image)

    if type(input_label) is np.ndarray:
        input_label = tf.convert_to_tensor(input_label)

    # random initialization around input_image
    random_noise = tf.random.uniform(shape=input_image.shape, minval=-epsilon, maxval=epsilon)
    adv_image = input_image + random_noise
    adv_noise = random_noise

    for _ in range(num_steps):
        with tf.GradientTape(watch_accessed_variables=False) as tape:
            tape.watch(adv_image)
            adv_output = model(adv_image)
            if not soft_label:
                loss = loss_fn(input_label, adv_output, from_logits=from_logits) # use ground-truth label to attack
            else:
                pred_label = tf.math.argmax(adv_output, axis=1)
                loss = loss_fn(pred_label, adv_output, from_logits=from_logits) # use predicted label to attack

        gradient = tape.gradient(loss, adv_image) # get the gradient of the loss w.r.t. the current point
        adv_image_new = adv_image + step_size * tf.sign(gradient) # move current adverarial example along the gradient direction with step size is eta
        adv_image_new = tf.clip_by_value(adv_image_new, input_image-epsilon, input_image+epsilon) # clip to a valid boundary
        adv_image_new = tf.clip_by_value(adv_image_new, clip_value_min, clip_value_max) # clip to a valid range
        adv_noise = gamma*adv_noise + (1-gamma)*(adv_image_new - adv_image)
        adv_image = adv_image_new
        adv_image = tf.stop_gradient(adv_image) # stop the gradient to make the adversarial image as a constant input
    adv_image = adv_image + adv_noise
    adv_image = tf.clip_by_value(adv_image, input_image-epsilon, input_image+epsilon) # clip to a valid boundary
    adv_image = tf.clip_by_value(adv_image, clip_value_min, clip_value_max) # clip to a valid range
    return adv_image


# Load the saved model
from matplotlib import pyplot as plt
from tensorflow.keras.applications.vgg19 import preprocess_input, decode_predictions

loaded_model = load_model('models/data_mixup_model.h5')

def attack_pgd ():
    random_indices = np.random.choice(len(data_manager.X_test), size=20, replace=False)
    correct = 0
    for i in random_indices:
        x = np.expand_dims(data_manager.X_test[i], axis=0)
        x = tf.cast(x, dtype=tf.float32)

        x_pgd= pgd_attack(loaded_model, x, data_manager.y_test[i])
        preds = loaded_model.predict(x)
        pgd_pred = loaded_model.predict(x_pgd)
        true_label= data_manager.classes[np.argmax(preds)]
        adv_label= data_manager.classes[np.argmax(pgd_pred)]
        if true_label == adv_label:
            correct += 1
        img = data_manager.X_test[i]
        img_pgd = np.squeeze(x_pgd.numpy())
        noise_pgd = np.clip(np.abs(img_pgd - img)*20, 0, 255).astype('int') # we multiply the noise by 20 for visualization
        fig = plt.figure(figsize=(15, 15*3))
        shown_img = img_pgd

        for i in range(3):
            shown_img = img if i==0 else noise_pgd if i==1 else img_pgd
            shown_label = 'Original image: {}'.format(true_label) if i==0 else 'Noise' if i==1 else 'Adversarial image: {}'.format(adv_label)
            plt.subplot(1,3,i+1)
            plt.imshow(shown_img)
            plt.xlabel(shown_label, fontsize= 12)
            plt.xticks([])
            plt.yticks([])
            plt.grid(False)
    return correct / 20 if correct > 0 else 0
attack_pgd()

1/1 [==============================] - 0s 25ms/step
1/1 [==============================] - 0s 27ms/step

C:\Users\manut\anaconda3\envs\gpu\lib\site-packages\tensorflow\python\data\ops\structured_function.py:264: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.
  warnings.warn(
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 26ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 25ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 25ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 25ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).


def attack_fgsm ():
    random_indices = np.random.choice(len(data_manager.X_test), size=20, replace=False)
    correct = 0
    for i in random_indices:
        x = np.expand_dims(data_manager.X_test[i], axis=0)
        x = tf.cast(x, dtype=tf.float32)

        x_pgd= fgsm_attack(loaded_model, x, data_manager.y_test[i])
        preds = loaded_model.predict(x)
        pgd_pred = loaded_model.predict(x_pgd)
        true_label= data_manager.classes[np.argmax(preds)]
        adv_label= data_manager.classes[np.argmax(pgd_pred)]
        if true_label == adv_label:
            correct += 1
        img = data_manager.X_test[i]
        img_pgd = np.squeeze(x_pgd.numpy())
        noise_pgd = np.clip(np.abs(img_pgd - img)*20, 0, 255).astype('int') # we multiply the noise by 20 for visualization
        fig = plt.figure(figsize=(15, 15*3))
        shown_img = img_pgd

        for i in range(3):
            shown_img = img if i==0 else noise_pgd if i==1 else img_pgd
            shown_label = 'Original image: {}'.format(true_label) if i==0 else 'Noise' if i==1 else 'Adversarial image: {}'.format(adv_label)
            plt.subplot(1,3,i+1)
            plt.imshow(shown_img)
            plt.xlabel(shown_label, fontsize= 12)
            plt.xticks([])
            plt.yticks([])
            plt.grid(False)
    return correct / 20 if correct > 0 else 0
attack_fgsm()

1/1 [==============================] - 0s 36ms/step
1/1 [==============================] - 0s 34ms/step

C:\Users\manut\anaconda3\envs\gpu\lib\site-packages\keras\backend.py:5582: UserWarning: "`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a Softmax activation and thus does not represent logits. Was this intended?
  output, from_logits = _get_logits(
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 28ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 26ms/step
1/1 [==============================] - 0s 26ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 32ms/step
1/1 [==============================] - 0s 32ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 26ms/step
1/1 [==============================] - 0s 28ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).


def attack_mim ():
    random_indices = np.random.choice(len(data_manager.X_test), size=20, replace=False)
    correct = 0
    for i in random_indices:
        x = np.expand_dims(data_manager.X_test[i], axis=0)
        x = tf.cast(x, dtype=tf.float32)

        x_pgd= fgsm_attack(loaded_model, x, data_manager.y_test[i])
        preds = loaded_model.predict(x)
        pgd_pred = loaded_model.predict(x_pgd)
        true_label= data_manager.classes[np.argmax(preds)]
        adv_label= data_manager.classes[np.argmax(pgd_pred)]
        if true_label == adv_label:
            correct += 1
        img = data_manager.X_test[i]
        img_pgd = np.squeeze(x_pgd.numpy())
        noise_pgd = np.clip(np.abs(img_pgd - img)*20, 0, 255).astype('int') # we multiply the noise by 20 for visualization
        fig = plt.figure(figsize=(15, 15*3))
        shown_img = img_pgd

        for i in range(3):
            shown_img = img if i==0 else noise_pgd if i==1 else img_pgd
            shown_label = 'Original image: {}'.format(true_label) if i==0 else 'Noise' if i==1 else 'Adversarial image: {}'.format(adv_label)
            plt.subplot(1,3,i+1)
            plt.imshow(shown_img)
            plt.xlabel(shown_label, fontsize= 12)
            plt.xticks([])
            plt.yticks([])
            plt.grid(False)
    return correct / 20 if correct > 0 else 0
attack_mim()

1/1 [==============================] - 0s 26ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 30ms/step
1/1 [==============================] - 0s 33ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 26ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 27ms/step
1/1 [==============================] - 0s 28ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).

1/1 [==============================] - 0s 28ms/step
1/1 [==============================] - 0s 27ms/step

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).


from sklearn.metrics import accuracy_score
lenet_defence = loaded_model

#Insert your code here. You can add more cells if necessary
optimizer = tf.optimizers.Adam(learning_rate=0.001)
loss_obj = tf.nn.sparse_softmax_cross_entropy_with_logits

# metrics to track the different accuracies.
train_loss = tf.metrics.Mean(name='train_loss')
test_acc_clean = tf.metrics.SparseCategoricalAccuracy()
test_acc_pgd = tf.metrics.SparseCategoricalAccuracy()
batch_size = 32

def train_step_adv(x, x_adv, y):
    with tf.GradientTape() as tape:
        logits = lenet_defence(x)
        logits_adv = lenet_defence(x_adv)
        loss = (loss_obj(y, logits) + loss_obj(y, logits_adv))/2
        gradients = tape.gradient(loss, lenet_defence.trainable_variables)
        optimizer.apply_gradients(zip(gradients, lenet_defence.trainable_variables))
    return loss

epochs = 5 # number of epochs
for epoch in range(epochs):
    print ("\n")
    print ("epoch: ",epoch)
    # keras like display of progress
    progress_bar_train = tf.keras.utils.Progbar(data_manager.X_train.shape[0], verbose=1)
    for i in range (int (data_manager.X_train.shape[0]/ batch_size)):
        current_data = data_manager.next_batch(batch_size)
        (x,y) = current_data
        x = tf.cast(x, dtype=tf.float32)
        # train based on pgd attack according to the perimeters given 
        x_adv = pgd_attack(lenet_defence, x, y,  epsilon=0.0313,
              num_steps=10,
              step_size=0.002,
              clip_value_min=0.,
              clip_value_max=1.0,
              soft_label=False,
              from_logits= False)
        loss = train_step_adv(x, x_adv, y)
        y_pred = lenet_defence(x)
        test_acc_clean(y, y_pred)
        test_acc_pgd(y, lenet_defence(x_adv))
        train_loss(loss)
        progress_bar_train.add(x.shape[0], values=[('loss', train_loss.result()), ("acc (%)", test_acc_clean.result() * 100),("pgd (%)", test_acc_pgd.result() * 100)])
    print()


epoch:  0
7552/7560 [============================>.] - ETA: 0s - loss: 2.6384 - acc (%): 80.8675 - pgd (%): 17.7838


epoch:  1
7552/7560 [============================>.] - ETA: 0s - loss: 2.6217 - acc (%): 78.2272 - pgd (%): 22.1566


epoch:  2
7552/7560 [============================>.] - ETA: 0s - loss: 2.6073 - acc (%): 78.3158 - pgd (%): 24.6540


epoch:  3
7552/7560 [============================>.] - ETA: 0s - loss: 2.5977 - acc (%): 78.5100 - pgd (%): 26.2517


epoch:  4
7552/7560 [============================>.] - ETA: 0s - loss: 2.5896 - acc (%): 78.7079 - pgd (%): 27.5303


y_batch_adv = []
y_adv = []
y_true = []

for i in range(data_manager.X_test.shape[0] // batch_size):
    idx = data_manager.random.choice(data_manager.X_test.shape[0], batch_size, 
                                 replace=batch_size > data_manager.X_test.shape[0])
    x,y = data_manager.X_test[idx], data_manager.y_test[idx] 
    x_fgsm = fgsm_attack(lenet_defence, tf.cast(x, tf.float32), y, epsilon = 0.0313,
                           soft_label=False, clip_value_min=0.0, clip_value_max=255.0, from_logits=False)
    
    y_batch_adv = np.argmax(loaded_model(x_fgsm).numpy(), 1)
    y_adv.append(y_batch_adv[0].tolist())
    y_true.append(y[0].tolist())
    
    test_adv_acc = accuracy_score(y_true, y_adv)
print("FGSM attack accuracy:{}".format(test_adv_acc))

FGSM attack accuracy:0.3103448275862069


y_batch_adv = []
y_adv = []
y_true = []

for i in range(data_manager.X_test.shape[0] // batch_size):
    idx = data_manager.random.choice(data_manager.X_test.shape[0], batch_size, 
                                 replace=batch_size > data_manager.X_test.shape[0])
    x,y = data_manager.X_test[idx], data_manager.y_test[idx] 
    x_pgd = pgd_attack(lenet_defence, tf.cast(x, tf.float32), y, num_steps=20, step_size= 0.002, epsilon = 0.0313,
                   soft_label=False, clip_value_min=0.0, clip_value_max=255.0, from_logits=False)
    
    y_batch_adv = np.argmax(loaded_model(x_pgd).numpy(), 1)
    y_adv.append(y_batch_adv[0].tolist())
    y_true.append(y[0].tolist())
    
    test_adv_acc = accuracy_score(y_true, y_adv)
print("PGD attack accuracy:{}".format(test_adv_acc))

PGD attack accuracy:0.2413793103448276


y_batch_adv = []
y_adv = []
y_true = []

for i in range(data_manager.X_test.shape[0] // batch_size):
    idx = data_manager.random.choice(data_manager.X_test.shape[0], batch_size, 
                                 replace=batch_size > data_manager.X_test.shape[0])
    x,y = data_manager.X_test[idx], data_manager.y_test[idx] 
    x_mim = mim_attack(lenet_defence, tf.cast(x, tf.float32), y, num_steps=20, step_size= 0.002, epsilon = 0.0313,
                   soft_label=False, clip_value_min=0.0, clip_value_max=255.0, from_logits=False)
    
    y_batch_adv = np.argmax(loaded_model(x_mim).numpy(), 1)
    y_adv.append(y_batch_adv[0].tolist())
    y_true.append(y[0].tolist())
    
    test_adv_acc = accuracy_score(y_true, y_adv)
print("MIM attack accuracy:{}".format(test_adv_acc))

MIM attack accuracy:0.27586206896551724

FIT5215: Deep Learning (2023)¶

Student Information¶

Deep Neural Networks¶

Due: 11:59pm Sunday, 10 September 2023 (Sunday)¶

Important note: This is an individual assignment. It contributes 20% to your final mark. Read the assignment instruction carefully.¶

Instruction¶

What to submit¶

Part 1: Theory and Knowledge Questions¶

Question 1.1 Activation function plays an important role in modern Deep NNs. For each of the activation function below, state its output range, find its derivative (show your steps), and plot the activation fuction and its derivative¶

Question 1.2 Assume that we feed a data point $x$ with a ground-truth label $y=3$ to the feed-forward neural network with the `ReLU activation` function as shown in the following figure¶

Option 1¶

Option 2¶

Part 2: Deep Neural Networks (DNN) ¶

Question 2.1. Load the Fashion MNIST using Keras datasets¶

Question 2.2. Preprocess the dataset and split into training, validation, and testing datasets¶

Question 2.3. Write code for the feed-forward neural net using TF 2.x¶

Question 2.4. Tuning hyper-parameters with grid search¶

Question 2.5. Experimenting with sharpness-aware minimization technique¶

Part 3: Convolutional Neural Networks and Image Classification¶

Report the best parameters and the testing accuracy here¶

Write your answer and observation here¶

Write your answer and observation here¶

FIT5215: Deep Learning (2023)¶

Student Information¶

Deep Neural Networks¶

Due: 11:59pm Sunday, 10 September 2023 (Sunday)¶

Important note: This is an individual assignment. It contributes 20% to your final mark. Read the assignment instruction carefully.¶

Instruction¶

What to submit¶

Part 1: Theory and Knowledge Questions¶

**Question 1.1** Activation function plays an important role in modern Deep NNs. For each of the activation function below, state its output range, find its derivative (show your steps), and plot the activation fuction and its derivative¶

**Question 1.2** Assume that we feed a data point $x$ with a ground-truth label $y=3$ to the feed-forward neural network with the ReLU activation function as shown in the following figure¶

**Option 1**¶

**Option 2**¶

Part 2: Deep Neural Networks (DNN) ¶

**Question 2.1**. Load the Fashion MNIST using Keras datasets¶

**Question 2.2**. Preprocess the dataset and split into training, validation, and testing datasets¶

**Question 2.3**. Write code for the feed-forward neural net using TF 2.x¶

**Question 2.4**. Tuning hyper-parameters with grid search¶

**Question 2.5**. Experimenting with sharpness-aware minimization technique¶

Part 3: Convolutional Neural Networks and Image Classification¶

Report the best parameters and the testing accuracy here¶

Write your answer and observation here¶

Write your answer and observation here¶

Question 1.1 Activation function plays an important role in modern Deep NNs. For each of the activation function below, state its output range, find its derivative (show your steps), and plot the activation fuction and its derivative¶

Question 1.2 Assume that we feed a data point $x$ with a ground-truth label $y=3$ to the feed-forward neural network with the `ReLU activation` function as shown in the following figure¶

Option 1¶

Option 2¶

Question 2.1. Load the Fashion MNIST using Keras datasets¶

Question 2.2. Preprocess the dataset and split into training, validation, and testing datasets¶

Question 2.3. Write code for the feed-forward neural net using TF 2.x¶

Question 2.4. Tuning hyper-parameters with grid search¶

Question 2.5. Experimenting with sharpness-aware minimization technique¶